zink: add renderdoc handling

renderdoc won't work with zink in frontends that aren't dri,
so ZINK_RENDERDOC should be used to specify start:end frames
to ensure that the vulkan command stream is captured

this is not a renderdoc issue: there are no frame boundaries in rusticl
or gallium-nine, so there is no possible way that renderdoc could
determine when/how to split frames

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20651>

meson.build

@@ -1438,7 +1438,8 @@ if not ['linux'].contains(host_machine.system())
   endif
 endif
 
-foreach h : ['xlocale.h', 'linux/futex.h', 'endian.h', 'dlfcn.h', 'sys/shm.h', 'cet.h', 'pthread_np.h']
+foreach h : ['xlocale.h', 'linux/futex.h', 'endian.h', 'dlfcn.h', 'sys/shm.h',
+             'cet.h', 'pthread_np.h', 'renderdoc_app.h']
   if cc.check_header(h)
     pre_args += '-DHAVE_@0@'.format(h.to_upper().underscorify())
   endif

src/gallium/drivers/zink/zink_batch.c

@@ -437,6 +437,26 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch)
       batch->last_batch_usage = &last_state->usage;
    }
 
+#ifdef HAVE_RENDERDOC_APP_H
+   if (VKCTX(CmdInsertDebugUtilsLabelEXT) && screen->renderdoc_api) {
+      VkDebugUtilsLabelEXT capture_label;
+      /* Magic fallback which lets us bridge the Wine barrier over to Linux RenderDoc. */
+      capture_label.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT;
+      capture_label.pNext = NULL;
+      capture_label.pLabelName = "vr-marker,frame_end,type,application";
+      memset(capture_label.color, 0, sizeof(capture_label.color));
+      VKCTX(CmdInsertDebugUtilsLabelEXT)(batch->state->barrier_cmdbuf, &capture_label);
+      VKCTX(CmdInsertDebugUtilsLabelEXT)(batch->state->cmdbuf, &capture_label);
+   }
+
+   unsigned renderdoc_frame = p_atomic_read(&screen->renderdoc_frame);
+   if (!(ctx->flags & ZINK_CONTEXT_COPY_ONLY) && screen->renderdoc_api && !screen->renderdoc_capturing &&
+        ((screen->renderdoc_capture_all && screen->screen_id == 1) || (renderdoc_frame >= screen->renderdoc_capture_start && renderdoc_frame <= screen->renderdoc_capture_end))) {
+      screen->renderdoc_api->StartFrameCapture(RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(screen->instance), NULL);
+      screen->renderdoc_capturing = true;
+   }
+#endif
+
    if (!ctx->queries_disabled)
       zink_resume_queries(ctx, batch);
 
@@ -635,6 +655,12 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch)
       submit_queue(bs, NULL, 0);
       post_submit(bs, NULL, 0);
    }
+#ifdef HAVE_RENDERDOC_APP_H
+   if (!(ctx->flags & ZINK_CONTEXT_COPY_ONLY) && screen->renderdoc_capturing && p_atomic_read(&screen->renderdoc_frame) > screen->renderdoc_capture_end) {
+      screen->renderdoc_api->EndFrameCapture(RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(screen->instance), NULL);
+      screen->renderdoc_capturing = false;
+   }
+#endif
 }
 
 static int

src/gallium/drivers/zink/zink_context.c

@@ -3730,8 +3730,11 @@ zink_flush(struct pipe_context *pctx,
       ctx->rp_changed |= fbfetch_outputs > 0;
    }
 
-   if (ctx->needs_present && (flags & PIPE_FLUSH_END_OF_FRAME)) {
-      if (ctx->needs_present->obj->image)
+   if (flags & PIPE_FLUSH_END_OF_FRAME) {
+#ifdef HAVE_RENDERDOC_APP_H
+      p_atomic_inc(&screen->renderdoc_frame);
+#endif
+      if (ctx->needs_present && ctx->needs_present->obj->image)
          zink_screen(ctx->base.screen)->image_barrier(ctx, ctx->needs_present, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
       ctx->needs_present = NULL;
    }

src/gallium/drivers/zink/zink_screen.c

@@ -49,6 +49,8 @@
 
 #include "util/u_cpu_detect.h"
 
+static int num_screens = 0;
+
 #if DETECT_OS_WINDOWS
 #include <io.h>
 #define VK_LIBNAME "vulkan-1.dll"
@@ -1384,6 +1386,11 @@ zink_destroy_screen(struct pipe_screen *pscreen)
 {
    struct zink_screen *screen = zink_screen(pscreen);
 
+#ifdef HAVE_RENDERDOC_APP_H
+   if (screen->renderdoc_capture_all && p_atomic_dec_zero(&num_screens))
+      screen->renderdoc_api->EndFrameCapture(RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(screen->instance), NULL);
+#endif
+
    hash_table_foreach(&screen->dts, entry)
       zink_kopper_deinit_displaytarget(screen, entry->data);
    simple_mtx_destroy(&screen->dt_lock);
@@ -1977,6 +1984,44 @@ populate_format_props(struct zink_screen *screen)
       screen->need_2D_sparse = !screen->base.get_sparse_texture_virtual_page_size(&screen->base, PIPE_TEXTURE_1D, false, PIPE_FORMAT_R32_FLOAT, 0, 16, NULL, NULL, NULL);
 }
 
+static void
+setup_renderdoc(struct zink_screen *screen)
+{
+#ifdef HAVE_RENDERDOC_APP_H
+   void *renderdoc = dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
+   /* not loaded */
+   if (!renderdoc)
+      return;
+
+   pRENDERDOC_GetAPI get_api = dlsym(renderdoc, "RENDERDOC_GetAPI");
+   if (!get_api)
+      return;
+
+   /* need synchronous dispatch for renderdoc coherency */
+   screen->threaded = false;
+   get_api(eRENDERDOC_API_Version_1_0_0, (void*)&screen->renderdoc_api);
+   screen->renderdoc_api->SetActiveWindow(RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(screen->instance), NULL);
+
+   const char *capture_id = debug_get_option("ZINK_RENDERDOC", NULL);
+   if (!capture_id)
+      return;
+   int count = sscanf(capture_id, "%u:%u", &screen->renderdoc_capture_start, &screen->renderdoc_capture_end);
+   if (count != 2) {
+      count = sscanf(capture_id, "%u", &screen->renderdoc_capture_start);
+      if (!count) {
+         if (!strcmp(capture_id, "all")) {
+            screen->renderdoc_capture_all = true;
+         } else {
+            printf("`ZINK_RENDERDOC` usage: ZINK_RENDERDOC=all|frame_no[:end_frame_no]\n");
+            abort();
+         }
+      }
+      screen->renderdoc_capture_end = screen->renderdoc_capture_start;
+   }
+   p_atomic_set(&screen->renderdoc_frame, 1);
+#endif
+}
+
 bool
 zink_screen_init_semaphore(struct zink_screen *screen)
 {
@@ -2536,6 +2581,7 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
       goto fail;
    }
 
+   setup_renderdoc(screen);
    if (screen->threaded && !util_queue_init(&screen->flush_queue, "zfq", 8, 1, UTIL_QUEUE_INIT_RESIZE_IF_FULL, screen)) {
       mesa_loge("zink: Failed to create flush queue.\n");
       goto fail;
@@ -2806,6 +2852,8 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
    if (!screen->optimal_keys)
       screen->info.have_EXT_graphics_pipeline_library = false;
 
+   screen->screen_id = p_atomic_inc_return(&num_screens);
+
    return screen;
 
 fail:

src/gallium/drivers/zink/zink_types.h

@@ -61,6 +61,9 @@
 #include "zink_shader_keys.h"
 #include "vk_dispatch_table.h"
 
+#ifdef HAVE_RENDERDOC_APP_H
+#include "renderdoc_app.h"
+#endif
 
 /* the descriptor binding id for fbfetch/input attachment */
 #define ZINK_FBFETCH_BINDING 5
@@ -1295,6 +1298,17 @@ struct zink_screen {
 
    uint32_t cur_custom_border_color_samplers;
 
+   unsigned screen_id;
+
+#ifdef HAVE_RENDERDOC_APP_H
+   RENDERDOC_API_1_0_0 *renderdoc_api;
+   unsigned renderdoc_capture_start;
+   unsigned renderdoc_capture_end;
+   unsigned renderdoc_frame;
+   bool renderdoc_capturing;
+   bool renderdoc_capture_all;
+#endif
+
    struct vk_dispatch_table vk;
 
    void (*buffer_barrier)(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline);