qt: check for frameless window hint when probing Win11 22H2 window backdrop effect

Backdrop effect needs the interface window to be translucent, as we want the DWM
composited backdrop effect to pass through.

It is additionally required to use `Qt::FramelessWindowHint` for transparent
windows on Windows, otherwise artifacts may occur. Since this flag is explicitly
not used on Windows unlike other platforms, this change effectively means that
the native effect is not going to be used.

It is not clear why `Qt::FramelessWindowHint` is not used on Windows when CSD
is in use, because it is the canonical way of making a QWindow frameless.

modules/gui/qt/maininterface/compositor.cpp

@@ -381,7 +381,7 @@ bool CompositorVideo::setBlurBehind(QWindow *window, const bool enable)
         }
     }
 
-    if (!m_windowEffectsModule->isEffectAvailable(WindowEffectsModule::BlurBehind))
+    if (!m_windowEffectsModule->isEffectAvailable(window, WindowEffectsModule::BlurBehind))
         return false;
 
     m_windowEffectsModule->setBlurBehind(window, enable);

modules/gui/qt/maininterface/kwindowsystem_module.cpp

@@ -23,7 +23,7 @@
 
 #include <KWindowEffects>
 
-static bool isEffectAvailable(const WindowEffectsModule::Effect effect)
+static bool isEffectAvailable(const QWindow*, const WindowEffectsModule::Effect effect)
 {
     KWindowEffects::Effect kWindowEffect;
 
@@ -54,7 +54,7 @@ static int Open(vlc_object_t* const p_this)
     // In that case, simply fail here,
     // so that another potentially compatible
     // module can be loaded instead:
-    if (!isEffectAvailable(WindowEffectsModule::BlurBehind))
+    if (!isEffectAvailable(nullptr, WindowEffectsModule::BlurBehind))
         return VLC_EGENERIC;
 
     const auto obj = reinterpret_cast<WindowEffectsModule*>(p_this);

modules/gui/qt/maininterface/win32windoweffects_module.cpp

@@ -25,12 +25,34 @@
 
 #include <dwmapi.h>
 
-static bool isEffectAvailable(const WindowEffectsModule::Effect effect)
+static bool isEffectAvailable(const QWindow* window, const WindowEffectsModule::Effect effect)
 {
     // Version check is done on module open, no need to re-do it here.
     switch (effect)
     {
     case WindowEffectsModule::BlurBehind:
+        // NOTE: Qt does not officially support translucent window with frame.
+        //       The documentation states that `Qt::FramelessWindowHint` is
+        //       required on certain platforms, such as Windows. Otherwise,
+        //       The window starts with a white background, which is a widely
+        //       known Windows issue regardless of translucency, but the white
+        //       background is never cleared if the window clear color is
+        //       translucent. In this case, minimizing and restoring the window
+        //       makes the background cleared, but this still does not make
+        //       it a portable solution.
+        // NOTE: See QTBUG-56201, QTBUG-120691. From the reports, it appears
+        //       that Nvidia graphics is "fine" with translucent framed window
+        //       while Intel graphics is not. However, the said issue above
+        //       is still a concern with Nvidia graphics according to my own
+        //       experience.
+        // TODO: Ideally, we should at least use the frameless window hint
+        //       when CSD is in use and use native backdrop effect since
+        //       the custom solution has more chance to cause issues.
+        if (!window->flags().testFlag(Qt::FramelessWindowHint))
+        {
+            qDebug("Target window is not frameless, window can not be translucent for the Windows 11 22H2 native acrylic backdrop effect.");
+            return false;
+        }
         return true;
     default:
         return false;

modules/gui/qt/maininterface/windoweffects_module.hpp

@@ -33,7 +33,7 @@ struct WindowEffectsModule
     module_t *p_module = nullptr;
     void *p_sys = nullptr;
 
-    bool (*isEffectAvailable)(Effect effect);
+    bool (*isEffectAvailable)(const QWindow* window, Effect effect);
     void (*setBlurBehind)(QWindow* window, bool enable);
 };