diff --git a/src/api/protocol.rs b/src/api/protocol.rs
index 04621ac..23b0fee 100644
--- a/src/api/protocol.rs
+++ b/src/api/protocol.rs
@@ -1454,18 +1454,24 @@ impl pinnacle_api_defs::pinnacle::window::v0alpha1::window_service_server::Windo
             let Some(tag) = tag_id.tag(state) else { return };
 
             // TODO: turn state.tags into a hashset
-            window.with_state(|state| state.tags.retain(|tg| tg != &tag));
             match set_or_toggle {
                 Some(set) => {
                     if set {
                         window.with_state(|state| {
+                            state.tags.retain(|tg| tg != &tag);
                             state.tags.push(tag.clone());
                         })
+                    } else {
+                        window.with_state(|state| {
+                            state.tags.retain(|tg| tg != &tag);
+                        })
                     }
                 }
                 None => window.with_state(|state| {
                     if !state.tags.contains(&tag) {
                         state.tags.push(tag.clone());
+                    } else {
+                        state.tags.retain(|tg| tg != &tag);
                     }
                 }),
             }
diff --git a/src/state.rs b/src/state.rs
index fff4cba..80f91b5 100644
--- a/src/state.rs
+++ b/src/state.rs
@@ -1,28 +1,13 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 use crate::{
-    api::{
-        msg::Msg,
-        protocol::{
-            InputService, OutputService, PinnacleService, ProcessService, TagService, WindowService,
-        },
-        ApiState,
-    },
+    api::{msg::Msg, ApiState},
     backend::Backend,
     config::Config,
     cursor::Cursor,
     focus::FocusState,
     grab::resize_grab::ResizeSurfaceState,
     window::WindowElement,
-    XDG_BASE_DIRS,
-};
-use pinnacle_api_defs::pinnacle::v0alpha1::pinnacle_service_server::PinnacleServiceServer;
-use pinnacle_api_defs::pinnacle::{
-    input::v0alpha1::input_service_server::InputServiceServer,
-    output::v0alpha1::output_service_server::OutputServiceServer,
-    process::v0alpha1::process_service_server::ProcessServiceServer,
-    tag::v0alpha1::tag_service_server::TagServiceServer,
-    window::v0alpha1::window_service_server::WindowServiceServer,
 };
 use smithay::{
     desktop::{PopupManager, Space},