(Re)workaround cursor overwriting transparency in screencopy

Cargo.lock

@@ -2117,7 +2117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
 dependencies = [
  "cfg-if",
- "windows-targets 0.52.5",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]

src/backend/udev.rs

@@ -32,7 +32,7 @@ use smithay::{
         libinput::{LibinputInputBackend, LibinputSessionInterface},
         renderer::{
             self, damage,
-            element::{self, surface::render_elements_from_surface_tree, Element},
+            element::{self, surface::render_elements_from_surface_tree, Element, Id},
             gles::{GlesRenderbuffer, GlesRenderer},
             multigpu::{gbm::GbmGlesBackend, GpuManager, MultiRenderer},
             sync::SyncPoint,
@@ -1454,7 +1454,20 @@ impl Udev {
             || (pinnacle.lock_state.is_locked()
                 && output.with_state(|state| state.lock_surface.is_none()));
 
-        let pointer_render_elements = pointer_render_elements(
+        // HACK: Doing `blit_frame_result` with something on the cursor/overlay plane overwrites
+        // transparency. This workaround makes the cursor not be on the cursor plane for blitting.
+        let kind = if output.with_state(|state| {
+            state
+                .screencopy
+                .as_ref()
+                .is_some_and(|sc| sc.overlay_cursor())
+        }) {
+            element::Kind::Unspecified
+        } else {
+            element::Kind::Cursor
+        };
+
+        let (pointer_render_elements, cursor_ids) = pointer_render_elements(
             output,
             &mut renderer,
             &mut pinnacle.cursor_state,
@@ -1462,6 +1475,7 @@ impl Udev {
             pointer_location,
             pinnacle.dnd_icon.as_ref(),
             &pinnacle.clock,
+            kind,
         );
         output_render_elements.extend(
             pointer_render_elements
@@ -1543,6 +1557,7 @@ impl Udev {
                     surface,
                     &render_frame_result,
                     &pinnacle.loop_handle,
+                    cursor_ids,
                 );
             }
 
@@ -1610,6 +1625,7 @@ fn handle_pending_screencopy<'a>(
     surface: &mut RenderSurface,
     render_frame_result: &UdevRenderFrameResult<'a>,
     loop_handle: &LoopHandle<'static, State>,
+    cursor_ids: Vec<Id>,
 ) {
     let Some(mut screencopy) = output.with_state_mut(|state| state.screencopy.take()) else {
         return;
@@ -1733,7 +1749,7 @@ fn handle_pending_screencopy<'a>(
                     output.current_scale().fractional_scale(),
                     renderer,
                     [screencopy.physical_region()],
-                    [],
+                    cursor_ids,
                 )?))
             } else {
                 // `RenderFrameResult::blit_frame_result` doesn't expose a way to
@@ -1760,7 +1776,11 @@ fn handle_pending_screencopy<'a>(
                         Point::from((0, 0)),
                         untransformed_output_size,
                     )],
-                    [],
+                    if !screencopy.overlay_cursor() {
+                        cursor_ids
+                    } else {
+                        Vec::new()
+                    },
                 )?;
 
                 // ayo are we supposed to wait this here (granted it doesn't do anything
@@ -1840,7 +1860,11 @@ fn handle_pending_screencopy<'a>(
                         Point::from((0, 0)),
                         untransformed_output_size,
                     )],
-                    [],
+                    if !screencopy.overlay_cursor() {
+                        cursor_ids
+                    } else {
+                        Vec::new()
+                    },
                 )?;
 
                 // Can someone explain to me why it feels like some things are

src/backend/winit.rs

@@ -295,7 +295,7 @@ impl Winit {
                 .map(|ptr| ptr.current_location())
                 .unwrap_or((0.0, 0.0).into());
 
-            let pointer_render_elements = pointer_render_elements(
+            let (pointer_render_elements, _cursor_ids) = pointer_render_elements(
                 &self.output,
                 self.backend.renderer(),
                 &mut pinnacle.cursor_state,
@@ -303,6 +303,7 @@ impl Winit {
                 pointer_location,
                 pinnacle.dnd_icon.as_ref(),
                 &pinnacle.clock,
+                element::Kind::Cursor,
             );
             output_render_elements.extend(
                 pointer_render_elements

src/cursor.rs

@@ -84,7 +84,9 @@ impl CursorState {
                 // TODO: scale
                 let buffer = MemoryRenderBuffer::from_slice(
                     &image.pixels_rgba,
-                    Fourcc::Abgr8888,
+                    // Don't make Abgr, then the format doesn't match the
+                    // cursor bo and this doesn't get put on the cursor plane
+                    Fourcc::Argb8888,
                     (image.width as i32, image.height as i32),
                     scale,
                     Transform::Normal,

src/render/pointer.rs

@@ -8,7 +8,7 @@ use smithay::{
             self,
             memory::MemoryRenderBufferRenderElement,
             surface::{render_elements_from_surface_tree, WaylandSurfaceRenderElement},
-            AsRenderElements,
+            AsRenderElements, Element, Id,
         },
         ImportAll, ImportMem,
     },
@@ -41,6 +41,9 @@ render_elements! {
     Memory = MemoryRenderBufferRenderElement<R>,
 }
 
+/// Render pointer elements.
+///
+/// Additionally returns the ids of cursor elements for use in screencopy.
 pub fn pointer_render_elements<R: PRenderer>(
     output: &Output,
     renderer: &mut R,
@@ -49,11 +52,13 @@ pub fn pointer_render_elements<R: PRenderer>(
     pointer_location: Point<f64, Logical>,
     dnd_icon: Option<&WlSurface>,
     clock: &Clock<Monotonic>,
-) -> Vec<PointerRenderElement<R>> {
+    kind: element::Kind,
+) -> (Vec<PointerRenderElement<R>>, Vec<Id>) {
     let mut pointer_render_elements = Vec::new();
+    let mut cursor_ids = Vec::new();
 
     let Some(output_geometry) = space.output_geometry(output) else {
-        return pointer_render_elements;
+        return (pointer_render_elements, cursor_ids);
     };
 
     let scale = Scale::from(output.current_scale().fractional_scale());
@@ -78,7 +83,7 @@ pub fn pointer_render_elements<R: PRenderer>(
                     None,
                     None,
                     None,
-                    element::Kind::Cursor,
+                    kind,
                 );
 
                 elem.map(|elem| vec![PointerRenderElement::Memory(elem)])
@@ -108,6 +113,9 @@ pub fn pointer_render_elements<R: PRenderer>(
             }
         };
 
+        // rust analyzer is so broken wtf why is `elem` {unknown}
+        cursor_ids = elements.iter().map(|elem| elem.id()).cloned().collect();
+
         if let Some(dnd_icon) = dnd_icon {
             elements.extend(AsRenderElements::render_elements(
                 &smithay::desktop::space::SurfaceTree::from_surface(dnd_icon),
@@ -121,5 +129,5 @@ pub fn pointer_render_elements<R: PRenderer>(
         pointer_render_elements = elements;
     }
 
-    pointer_render_elements
+    (pointer_render_elements, cursor_ids)
 }