radeonsi: add an option for debugging VM faults

Reviewed-by: Michel Dänzer <michel.daenzer@amd.com>

src/gallium/drivers/radeon/r600_pipe_common.c

@@ -359,6 +359,7 @@ static const struct debug_named_value common_debug_options[] = {
 	{ "forcedma", DBG_FORCE_DMA, "Use asynchronous DMA for all operations when possible." },
 	{ "precompile", DBG_PRECOMPILE, "Compile one shader variant at shader creation." },
 	{ "nowc", DBG_NO_WC, "Disable GTT write combining" },
+	{ "check_vm", DBG_CHECK_VM, "Check VM faults and dump debug info." },
 
 	DEBUG_NAMED_VALUE_END /* must be last */
 };

src/gallium/drivers/radeon/r600_pipe_common.h

@@ -98,6 +98,7 @@
 #define DBG_PRECOMPILE		(1llu << 39)
 #define DBG_INFO		(1llu << 40)
 #define DBG_NO_WC		(1llu << 41)
+#define DBG_CHECK_VM		(1llu << 42)
 
 #define R600_MAP_BUFFER_ALIGNMENT 64
 

src/gallium/drivers/radeonsi/si_debug.c

@@ -28,6 +28,7 @@
 #include "si_shader.h"
 #include "sid.h"
 #include "sid_tables.h"
+#include "ddebug/dd_util.h"
 
 
 static void si_dump_shader(struct si_shader_selector *sel, const char *name,
@@ -438,7 +439,119 @@ static void si_dump_debug_state(struct pipe_context *ctx, FILE *f,
 	fprintf(f, "Done.\n");
 }
 
+static bool si_vm_fault_occured(struct si_context *sctx, uint32_t *out_addr)
+{
+	char line[2000];
+	unsigned sec, usec;
+	int progress = 0;
+	uint64_t timestamp = 0;
+	bool fault = false;
+
+	FILE *p = popen("dmesg", "r");
+	if (!p)
+		return false;
+
+	while (fgets(line, sizeof(line), p)) {
+		char *msg, len;
+
+		/* Get the timestamp. */
+		if (sscanf(line, "[%u.%u]", &sec, &usec) != 2) {
+			assert(0);
+			continue;
+		}
+		timestamp = sec * 1000000llu + usec;
+
+		/* If just updating the timestamp. */
+		if (!out_addr)
+			continue;
+
+		/* Process messages only if the timestamp is newer. */
+		if (timestamp <= sctx->dmesg_timestamp)
+			continue;
+
+		/* Only process the first VM fault. */
+		if (fault)
+			continue;
+
+		/* Remove trailing \n */
+		len = strlen(line);
+		if (len && line[len-1] == '\n')
+			line[len-1] = 0;
+
+		/* Get the message part. */
+		msg = strchr(line, ']');
+		if (!msg) {
+			assert(0);
+			continue;
+		}
+		msg++;
+
+		switch (progress) {
+		case 0:
+			if (strstr(msg, "GPU fault detected:"))
+				progress = 1;
+			break;
+		case 1:
+			msg = strstr(msg, "VM_CONTEXT1_PROTECTION_FAULT_ADDR");
+			if (msg) {
+				msg = strstr(msg, "0x");
+				if (msg) {
+					msg += 2;
+					if (sscanf(msg, "%X", out_addr) == 1)
+						fault = true;
+				}
+			}
+			progress = 0;
+			break;
+		default:
+			progress = 0;
+		}
+	}
+	pclose(p);
+
+	if (timestamp > sctx->dmesg_timestamp)
+		sctx->dmesg_timestamp = timestamp;
+	return fault;
+}
+
+void si_check_vm_faults(struct si_context *sctx)
+{
+	struct pipe_screen *screen = sctx->b.b.screen;
+	FILE *f;
+	uint32_t addr;
+
+	/* Use conservative timeout 800ms, after which we won't wait any
+	 * longer and assume the GPU is hung.
+	 */
+	screen->fence_finish(screen, sctx->last_gfx_fence, 800*1000*1000);
+
+	if (!si_vm_fault_occured(sctx, &addr))
+		return;
+
+	f = dd_get_debug_file();
+	if (!f)
+		return;
+
+	fprintf(f, "VM fault report.\n\n");
+	fprintf(f, "Driver vendor: %s\n", screen->get_vendor(screen));
+	fprintf(f, "Device vendor: %s\n", screen->get_device_vendor(screen));
+	fprintf(f, "Device name: %s\n\n", screen->get_name(screen));
+	fprintf(f, "Failing VM page: 0x%08x\n\n", addr);
+
+	si_dump_last_ib(sctx, f);
+	fclose(f);
+
+	fprintf(stderr, "Detected a VM fault, exiting...\n");
+	exit(0);
+}
+
 void si_init_debug_functions(struct si_context *sctx)
 {
 	sctx->b.b.dump_debug_state = si_dump_debug_state;
+
+	/* Set the initial dmesg timestamp for this context, so that
+	 * only new messages will be checked for VM faults.
+	 */
+	if (sctx->screen->b.debug_flags & DBG_CHECK_VM)
+		si_vm_fault_occured(sctx, NULL);
 }

src/gallium/drivers/radeonsi/si_hw_context.c

@@ -103,6 +103,10 @@ void si_context_gfx_flush(void *context, unsigned flags,
 	if (fence)
 		ws->fence_reference(fence, ctx->last_gfx_fence);
 
+	/* Check VM faults if needed. */
+	if (ctx->screen->b.debug_flags & DBG_CHECK_VM)
+		si_check_vm_faults(ctx);
+
 	si_begin_new_cs(ctx);
 }
 

src/gallium/drivers/radeonsi/si_pipe.c

@@ -107,6 +107,9 @@ static struct pipe_context *si_create_context(struct pipe_screen *screen,
 	if (sctx == NULL)
 		return NULL;
 
+	if (sscreen->b.debug_flags & DBG_CHECK_VM)
+		flags |= PIPE_CONTEXT_DEBUG;
+
 	sctx->b.b.screen = screen; /* this must be set first */
 	sctx->b.b.priv = priv;
 	sctx->b.b.destroy = si_destroy_context;

src/gallium/drivers/radeonsi/si_pipe.h

@@ -276,6 +276,7 @@ struct si_context {
 	struct r600_resource	*last_trace_buf;
 	struct r600_resource	*trace_buf;
 	unsigned		trace_id;
+	uint64_t		dmesg_timestamp;
 };
 
 /* cik_sdma.c */
@@ -310,6 +311,7 @@ void si_init_cp_dma_functions(struct si_context *sctx);
 
 /* si_debug.c */
 void si_init_debug_functions(struct si_context *sctx);
+void si_check_vm_faults(struct si_context *sctx);
 
 /* si_dma.c */
 void si_dma_copy(struct pipe_context *ctx,