diff --git a/src/grab/move_grab.rs b/src/grab/move_grab.rs
index 56bc949..6af7a4a 100644
--- a/src/grab/move_grab.rs
+++ b/src/grab/move_grab.rs
@@ -87,6 +87,10 @@ impl PointerGrab<State> for MoveSurfaceGrab {
                 .cloned();
 
             if let Some(window_under) = window_under {
+                if state.layout_state.pending_swap {
+                    return;
+                }
+
                 if window_under == self.window {
                     return;
                 }
diff --git a/src/layout.rs b/src/layout.rs
index 09cff74..60167a3 100644
--- a/src/layout.rs
+++ b/src/layout.rs
@@ -146,6 +146,7 @@ impl State {
             if let Some(output) = win1.output(self) {
                 self.request_layout(&output);
             }
+            self.layout_state.pending_swap = true;
         }
     }
 }
@@ -157,6 +158,7 @@ pub struct LayoutRequestId(pub u32);
 #[derive(Debug, Default)]
 pub struct LayoutState {
     pub layout_request_sender: Option<UnboundedSender<Result<LayoutResponse, Status>>>,
+    pub pending_swap: bool,
     id_maps: HashMap<Output, LayoutRequestId>,
     pending_requests: HashMap<Output, Vec<(LayoutRequestId, Vec<WindowElement>)>>,
     old_requests: HashMap<Output, HashSet<LayoutRequestId>>,
@@ -297,6 +299,8 @@ impl State {
 
         self.schedule_render(&output);
 
+        self.layout_state.pending_swap = false;
+
         Ok(())
     }
 }