[PATCH v2] tests/nvme: Add admin-passthru+reset race test

[Jonathan Derrick]

Adds a test which runs many formats and reset_controllers in parallel.
The intent is to expose timing holes in the controller state machine
which will lead to hung task timeouts and the controller becoming
unavailable.

Reported by https://bugzilla.kernel.org/show_bug.cgi?id=216354

Signed-off-by: Jonathan Derrick <jonathan.derrick at linux.dev>
---
I seem to have isolated the error mechanism for older kernels, but 6.2.0-rc2
reliably segfaults my QEMU instance (something else to look into) and I don't
have any 'real' hardware to test this on at the moment. It looks like several
passthru commands are able to enqueue prior/during/after resetting/connecting.

The issue seems to be very heavily timing related, so the loop in the header is
a lot more forceful in this approach.

As far as the loop goes, I've noticed it will typically repro immediately or
pass the whole test.

 tests/nvme/047     | 121 +++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/047.out |   2 +
 2 files changed, 123 insertions(+)
 create mode 100755 tests/nvme/047
 create mode 100644 tests/nvme/047.out

diff --git a/tests/nvme/047 b/tests/nvme/047
new file mode 100755
index 0000000..fb8609c
--- /dev/null
+++ b/tests/nvme/047
@@ -0,0 +1,121 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2022 Jonathan Derrick <jonathan.derrick at linux.dev>
+#
+# Test nvme reset controller during admin passthru
+#
+# Regression for issue reported by
+# https://bugzilla.kernel.org/show_bug.cgi?id=216354
+#
+# Simpler form:
+# for i in {1..50}; do
+#	nvme format -f /dev/nvme0n1 &
+#	echo 1 > /sys/block/nvme0n1/device/reset_controller &
+# done
+
+. tests/nvme/rc
+
+#restrict test to nvme-pci only
+nvme_trtype=pci
+
+DESCRIPTION="test nvme reset controller during admin passthru"
+QUICK=1
+CAN_BE_ZONED=1
+
+RUN_TIME=300
+RESET_PCIE=true
+
+requires() {
+	_nvme_requires
+}
+
+device_requires() {
+	_require_test_dev_is_nvme
+}
+
+remove_and_rescan() {
+	local pdev=$1
+	echo 1 > /sys/bus/pci/devices/"$pdev"/remove
+	echo 1 > /sys/bus/pci/rescan
+}
+
+test_device() {
+	echo "Running ${TEST_NAME}"
+
+	local pdev
+	local blkdev
+	local ctrldev
+	local sysfs
+	local max_timeout
+	local timeout
+	local timeleft
+	local start
+	local last_live
+	local i
+
+	pdev="$(_get_pci_dev_from_blkdev)"
+	blkdev="${TEST_DEV_SYSFS##*/}"
+	ctrldev="$(echo "$blkdev" | grep -Eo 'nvme[0-9]+')"
+	sysfs="/sys/block/$blkdev/device"
+	max_timeout=$(cat /proc/sys/kernel/hung_task_timeout_secs)
+	timeout=$((max_timeout * 3 / 4))
+
+	sleep 5
+
+	start=$SECONDS
+	while [[ $((SECONDS - start)) -le $RUN_TIME ]]; do
+		if [[ $(cat "$sysfs/state") == "live" ]]; then
+			last_live=$SECONDS
+		fi
+
+		# Failure case appears to stack up formats while controller is resetting/connecting
+		if [[ $(pgrep -cf "nvme format") -lt 100 ]]; then
+			for ((i=0; i<100; i++)); do
+				nvme format -f "$TEST_DEV" &
+				echo 1 > "$sysfs/reset_controller" &
+			done &> /dev/null
+		fi
+
+		# Might have failed probe, so reset and continue test
+		if [[ $((SECONDS - last_live)) -gt 10 && \
+		      ! -c "/dev/$ctrldev" && "$RESET_PCIE" == true ]]; then
+			{
+				echo 1 > /sys/bus/pci/devices/"$pdev"/remove
+				echo 1 > /sys/bus/pci/rescan
+			} &
+
+			timeleft=$((max_timeout - timeout))
+			sleep $((timeleft < 30 ? timeleft : 30))
+			if [[ ! -c "/dev/$ctrldev" ]]; then
+				echo "/dev/$ctrldev missing"
+				echo "failed to reset $ctrldev's pcie device $pdev"
+				break
+			fi
+			sleep 5
+			continue
+		fi
+
+		if [[ $((SECONDS - last_live)) -gt $timeout ]]; then
+			if [[ ! -c "/dev/$ctrldev" ]]; then
+				echo "/dev/$ctrldev missing"
+				break
+			fi
+
+			# Assume the controller is hung and unrecoverable
+			if [[ -f "$sysfs/state" ]]; then
+				echo "nvme controller hung ($(cat "$sysfs/state"))"
+				break
+			else
+				echo "nvme controller hung"
+				break
+			fi
+		fi
+	done
+
+	if [[ ! -c "/dev/$ctrldev" || $(cat "$sysfs/state") != "live" ]]; then
+		echo "nvme still not live after $((SECONDS - last_live)) seconds!"
+	fi
+	udevadm settle
+
+	echo "Test complete"
+}
diff --git a/tests/nvme/047.out b/tests/nvme/047.out
new file mode 100644
index 0000000..915d0a2
--- /dev/null
+++ b/tests/nvme/047.out
@@ -0,0 +1,2 @@
+Running nvme/047
+Test complete
-- 
2.27.0