radv: Wire up ac_gather_context_rolls

Reviewed-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/27322>

docs/envvars.rst

@@ -349,6 +349,9 @@ Core Mesa environment variables
       * - ``rra``
         - Radeon Raytracing Analyzer
         - ``RADV``
+      * - ``ctxroll``
+        - Context rolls
+        - ``RADV``
 
    - Creating RMV captures requires the ``scripts/setup.sh`` script in the
      Radeon Developer Tools folder to be run beforehand

src/amd/vulkan/layers/radv_ctx_roll_layer.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright © 2024 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "radv_private.h"
+
+VKAPI_ATTR VkResult VKAPI_CALL
+ctx_roll_QueuePresentKHR(VkQueue _queue, const VkPresentInfoKHR *pPresentInfo)
+{
+   RADV_FROM_HANDLE(radv_queue, queue, _queue);
+
+   simple_mtx_lock(&queue->device->ctx_roll_mtx);
+
+   if (queue->device->ctx_roll_file) {
+      fclose(queue->device->ctx_roll_file);
+      queue->device->ctx_roll_file = NULL;
+   }
+
+   simple_mtx_unlock(&queue->device->ctx_roll_mtx);
+
+   return queue->device->layer_dispatch.ctx_roll.QueuePresentKHR(_queue, pPresentInfo);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+ctx_roll_QueueSubmit2(VkQueue _queue, uint32_t submitCount, const VkSubmitInfo2 *pSubmits, VkFence _fence)
+{
+   RADV_FROM_HANDLE(radv_queue, queue, _queue);
+
+   simple_mtx_lock(&queue->device->ctx_roll_mtx);
+
+   if (queue->device->ctx_roll_file) {
+      for (uint32_t submit_index = 0; submit_index < submitCount; submit_index++) {
+         const VkSubmitInfo2 *submit = pSubmits + submit_index;
+         for (uint32_t i = 0; i < submit->commandBufferInfoCount; i++) {
+            RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, submit->pCommandBufferInfos[i].commandBuffer);
+            fprintf(queue->device->ctx_roll_file, "\n%s:\n", vk_object_base_name(&cmd_buffer->vk.base));
+            queue->device->ws->cs_dump(cmd_buffer->cs, queue->device->ctx_roll_file, NULL, 0,
+                                       RADV_CS_DUMP_TYPE_CTX_ROLLS);
+         }
+      }
+   }
+
+   simple_mtx_unlock(&queue->device->ctx_roll_mtx);
+
+   return queue->device->layer_dispatch.ctx_roll.QueueSubmit2(_queue, submitCount, pSubmits, _fence);
+}

src/amd/vulkan/meson.build

@@ -32,6 +32,7 @@ radv_entrypoints_gen_command += [
   '--device-prefix', 'sqtt',
   '--device-prefix', 'rra',
   '--device-prefix', 'rmv',
+  '--device-prefix', 'ctx_roll',
 
   # Application layer entrypoints
   '--device-prefix', 'metro_exodus',
@@ -49,6 +50,7 @@ radv_entrypoints = custom_target(
 
 libradv_files = files(
   'bvh/bvh.h',
+  'layers/radv_ctx_roll_layer.c',
   'layers/radv_metro_exodus.c',
   'layers/radv_rage2.c',
   'layers/radv_quantic_dream.c',

src/amd/vulkan/radv_debug.c

@@ -102,7 +102,7 @@ static void
 radv_dump_trace(const struct radv_device *device, struct radeon_cmdbuf *cs, FILE *f)
 {
    fprintf(f, "Trace ID: %x\n", *device->trace_id_ptr);
-   device->ws->cs_dump(cs, f, (const int *)device->trace_id_ptr, 2);
+   device->ws->cs_dump(cs, f, (const int *)device->trace_id_ptr, 2, RADV_CS_DUMP_TYPE_IBS);
 }
 
 static void

src/amd/vulkan/radv_device.c

@@ -598,6 +598,7 @@ init_dispatch_tables(struct radv_device *device, struct radv_physical_device *ph
    b.tables[RADV_RGP_DISPATCH_TABLE] = &device->layer_dispatch.rgp;
    b.tables[RADV_RRA_DISPATCH_TABLE] = &device->layer_dispatch.rra;
    b.tables[RADV_RMV_DISPATCH_TABLE] = &device->layer_dispatch.rmv;
+   b.tables[RADV_CTX_ROLL_DISPATCH_TABLE] = &device->layer_dispatch.ctx_roll;
 
    if (!strcmp(physical_device->instance->drirc.app_layer, "metroexodus")) {
       add_entrypoints(&b, &metro_exodus_device_entrypoints, RADV_APP_DISPATCH_TABLE);
@@ -618,6 +619,9 @@ init_dispatch_tables(struct radv_device *device, struct radv_physical_device *ph
       add_entrypoints(&b, &rmv_device_entrypoints, RADV_RMV_DISPATCH_TABLE);
 #endif
 
+   if (physical_device->instance->vk.trace_mode & RADV_TRACE_MODE_CTX_ROLLS)
+      add_entrypoints(&b, &ctx_roll_device_entrypoints, RADV_CTX_ROLL_DISPATCH_TABLE);
+
    add_entrypoints(&b, &radv_device_entrypoints, RADV_DISPATCH_TABLE_COUNT);
    add_entrypoints(&b, &wsi_device_entrypoints, RADV_DISPATCH_TABLE_COUNT);
    add_entrypoints(&b, &vk_common_device_entrypoints, RADV_DISPATCH_TABLE_COUNT);
@@ -643,6 +647,22 @@ capture_trace(VkQueue _queue)
    if (queue->device->instance->vk.trace_mode & RADV_TRACE_MODE_RGP)
       queue->device->sqtt_triggered = true;
 
+   if (queue->device->instance->vk.trace_mode & RADV_TRACE_MODE_CTX_ROLLS) {
+      char filename[2048];
+      time_t t = time(NULL);
+      struct tm now = *localtime(&t);
+      snprintf(filename, sizeof(filename), "/tmp/%s_%04d.%02d.%02d_%02d.%02d.%02d.ctxroll", util_get_process_name(),
+               1900 + now.tm_year, now.tm_mon + 1, now.tm_mday, now.tm_hour, now.tm_min, now.tm_sec);
+
+      simple_mtx_lock(&queue->device->ctx_roll_mtx);
+
+      queue->device->ctx_roll_file = fopen(filename, "w");
+      if (queue->device->ctx_roll_file)
+         fprintf(stderr, "radv: Writing context rolls to '%s'...\n", filename);
+
+      simple_mtx_unlock(&queue->device->ctx_roll_mtx);
+   }
+
    return result;
 }
 
@@ -725,6 +745,7 @@ radv_CreateDevice(VkPhysicalDevice physicalDevice, const VkDeviceCreateInfo *pCr
 
    device->instance = physical_device->instance;
    device->physical_device = physical_device;
+   simple_mtx_init(&device->ctx_roll_mtx, mtx_plain);
    simple_mtx_init(&device->trace_mtx, mtx_plain);
    simple_mtx_init(&device->pstate_mtx, mtx_plain);
    simple_mtx_init(&device->rt_handles_mtx, mtx_plain);
@@ -1109,6 +1130,7 @@ fail_queue:
 
    _mesa_hash_table_destroy(device->rt_handles, NULL);
 
+   simple_mtx_destroy(&device->ctx_roll_mtx);
    simple_mtx_destroy(&device->pstate_mtx);
    simple_mtx_destroy(&device->trace_mtx);
    simple_mtx_destroy(&device->rt_handles_mtx);
@@ -1171,6 +1193,7 @@ radv_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator)
    }
 
    mtx_destroy(&device->overallocation_mutex);
+   simple_mtx_destroy(&device->ctx_roll_mtx);
    simple_mtx_destroy(&device->pstate_mtx);
    simple_mtx_destroy(&device->trace_mtx);
    simple_mtx_destroy(&device->rt_handles_mtx);

src/amd/vulkan/radv_instance.c

@@ -116,6 +116,7 @@ radv_get_perftest_option_name(int id)
 static const struct debug_control trace_options[] = {
    {"rgp", RADV_TRACE_MODE_RGP},
    {"rra", RADV_TRACE_MODE_RRA},
+   {"ctxroll", RADV_TRACE_MODE_CTX_ROLLS},
    {NULL, 0},
 };
 

src/amd/vulkan/radv_private.h

@@ -352,6 +352,9 @@ enum radv_trace_mode {
 
    /** Radeon Raytracing Analyzer */
    RADV_TRACE_MODE_RRA = 1 << (VK_TRACE_MODE_COUNT + 1),
+
+   /** Gather context rolls of submitted command buffers */
+   RADV_TRACE_MODE_CTX_ROLLS = 1 << (VK_TRACE_MODE_COUNT + 2),
 };
 
 struct radv_instance {
@@ -983,6 +986,7 @@ enum radv_dispatch_table {
    RADV_RGP_DISPATCH_TABLE,
    RADV_RRA_DISPATCH_TABLE,
    RADV_RMV_DISPATCH_TABLE,
+   RADV_CTX_ROLL_DISPATCH_TABLE,
    RADV_DISPATCH_TABLE_COUNT,
 };
 
@@ -991,6 +995,7 @@ struct radv_layer_dispatch_tables {
    struct vk_device_dispatch_table rgp;
    struct vk_device_dispatch_table rra;
    struct vk_device_dispatch_table rmv;
+   struct vk_device_dispatch_table ctx_roll;
 };
 
 enum radv_buffer_robustness {
@@ -1179,6 +1184,9 @@ struct radv_device {
    /* Radeon Raytracing Analyzer trace. */
    struct radv_rra_trace_data rra_trace;
 
+   FILE *ctx_roll_file;
+   simple_mtx_t ctx_roll_mtx;
+
    /* Trap handler. */
    struct radv_shader *trap_handler_shader;
    struct radeon_winsys_bo *tma_bo; /* Trap Memory Address */

src/amd/vulkan/radv_radeon_winsys.h

@@ -228,6 +228,11 @@ struct radv_winsys_gpuvm_fault_info {
    uint32_t vmhub;
 };
 
+enum radv_cs_dump_type {
+   RADV_CS_DUMP_TYPE_IBS,
+   RADV_CS_DUMP_TYPE_CTX_ROLLS,
+};
+
 struct radeon_winsys {
    void (*destroy)(struct radeon_winsys *ws);
 
@@ -303,7 +308,8 @@ struct radeon_winsys {
    void (*cs_execute_ib)(struct radeon_cmdbuf *cs, struct radeon_winsys_bo *bo, const uint64_t offset,
                          const uint32_t cdw, const bool predicate);
 
-   void (*cs_dump)(struct radeon_cmdbuf *cs, FILE *file, const int *trace_ids, int trace_id_count);
+   void (*cs_dump)(struct radeon_cmdbuf *cs, FILE *file, const int *trace_ids, int trace_id_count,
+                   enum radv_cs_dump_type type);
 
    void (*dump_bo_ranges)(struct radeon_winsys *ws, FILE *file);
 

src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c

@@ -1372,7 +1372,8 @@ radv_amdgpu_winsys_get_cpu_addr(void *_cs, uint64_t addr)
 }
 
 static void
-radv_amdgpu_winsys_cs_dump(struct radeon_cmdbuf *_cs, FILE *file, const int *trace_ids, int trace_id_count)
+radv_amdgpu_winsys_cs_dump(struct radeon_cmdbuf *_cs, FILE *file, const int *trace_ids, int trace_id_count,
+                           enum radv_cs_dump_type type)
 {
    struct radv_amdgpu_cs *cs = (struct radv_amdgpu_cs *)_cs;
    struct radv_amdgpu_winsys *ws = cs->ws;
@@ -1381,9 +1382,19 @@ radv_amdgpu_winsys_cs_dump(struct radeon_cmdbuf *_cs, FILE *file, const int *tra
       struct radv_amdgpu_cs_ib_info ib_info = radv_amdgpu_cs_ib_to_info(cs, cs->ib_buffers[0]);
       void *ib = radv_amdgpu_winsys_get_cpu_addr(cs, ib_info.ib_mc_address);
       assert(ib);
-      ac_parse_ib(file, ib, cs->ib_buffers[0].cdw, trace_ids, trace_id_count, "main IB", ws->info.gfx_level,
-                  ws->info.family, cs->hw_ip, radv_amdgpu_winsys_get_cpu_addr, cs);
+
+      if (type == RADV_CS_DUMP_TYPE_IBS) {
+         ac_parse_ib(file, ib, cs->ib_buffers[0].cdw, trace_ids, trace_id_count, "main IB", ws->info.gfx_level,
+                     ws->info.family, cs->hw_ip, radv_amdgpu_winsys_get_cpu_addr, cs);
+      } else {
+         uint32_t *ib_dw = ib;
+         ac_gather_context_rolls(file, &ib_dw, &cs->ib_buffers[0].cdw, 1, &ws->info);
+      }
    } else {
+      uint32_t **ibs = type == RADV_CS_DUMP_TYPE_CTX_ROLLS ? malloc(cs->num_ib_buffers * sizeof(uint32_t *)) : NULL;
+      uint32_t *ib_dw_sizes =
+         type == RADV_CS_DUMP_TYPE_CTX_ROLLS ? malloc(cs->num_ib_buffers * sizeof(uint32_t)) : NULL;
+
       for (unsigned i = 0; i < cs->num_ib_buffers; i++) {
          struct radv_amdgpu_ib *ib = &cs->ib_buffers[i];
          char name[64];
@@ -1399,8 +1410,20 @@ radv_amdgpu_winsys_cs_dump(struct radeon_cmdbuf *_cs, FILE *file, const int *tra
             snprintf(name, sizeof(name), "main IB");
          }
 
-         ac_parse_ib(file, mapped, ib->cdw, trace_ids, trace_id_count, name, ws->info.gfx_level, ws->info.family,
-                     cs->hw_ip, NULL, NULL);
+         if (type == RADV_CS_DUMP_TYPE_IBS) {
+            ac_parse_ib(file, mapped, ib->cdw, trace_ids, trace_id_count, name, ws->info.gfx_level, ws->info.family,
+                        cs->hw_ip, NULL, NULL);
+         } else {
+            ibs[i] = mapped;
+            ib_dw_sizes[i] = ib->cdw;
+         }
+      }
+
+      if (type == RADV_CS_DUMP_TYPE_CTX_ROLLS) {
+         ac_gather_context_rolls(file, ibs, ib_dw_sizes, cs->num_ib_buffers, &ws->info);
+
+         free(ibs);
+         free(ib_dw_sizes);
       }
    }
 }