diff --git a/sway/input/input-manager.c b/sway/input/input-manager.c
index 1115ba5e..5f7dfb42 100644
--- a/sway/input/input-manager.c
+++ b/sway/input/input-manager.c
@@ -532,6 +532,18 @@ static void retranslate_keysyms(struct input_config *input_config) {
 			return;
 		}
 	}
+
+	for (int i = 0; i < config->input_type_configs->length; ++i) {
+		struct input_config *ic = config->input_type_configs->items[i];
+		if (ic->xkb_layout || ic->xkb_file) {
+			// this is the first config with xkb_layout or xkb_file
+			if (ic->identifier == input_config->identifier) {
+				translate_keysyms(ic);
+			}
+
+			return;
+		}
+	}
 }
 
 static void input_manager_configure_input(