diff --git a/Content.Client/UserInterface/Systems/Actions/ActionUIController.cs b/Content.Client/UserInterface/Systems/Actions/ActionUIController.cs
index 1dffeb8d2d..a6c1cfc94f 100644
--- a/Content.Client/UserInterface/Systems/Actions/ActionUIController.cs
+++ b/Content.Client/UserInterface/Systems/Actions/ActionUIController.cs
@@ -398,10 +398,6 @@ public sealed class ActionUIController : UIController, IOnStateChanged<GameplayS
     {
         QueueWindowUpdate();
 
-        // TODO ACTIONS allow buttons to persist across state applications
-        // Then we don't have to interrupt drags any time the buttons get rebuilt.
-        _menuDragHelper.EndDrag();
-
         if (_actionsSystem != null)
             _container?.SetActionData(_actionsSystem, _actions.ToArray());
     }
diff --git a/Content.Client/UserInterface/Systems/Actions/Controls/ActionButtonContainer.cs b/Content.Client/UserInterface/Systems/Actions/Controls/ActionButtonContainer.cs
index 38c08dc472..67b96d0330 100644
--- a/Content.Client/UserInterface/Systems/Actions/Controls/ActionButtonContainer.cs
+++ b/Content.Client/UserInterface/Systems/Actions/Controls/ActionButtonContainer.cs
@@ -28,14 +28,26 @@ public class ActionButtonContainer : GridContainer
         get => (ActionButton) GetChild(index);
     }
 
-    private void BuildActionButtons(int count)
+    public void SetActionData(ActionsSystem system, params EntityUid?[] actionTypes)
     {
+        var uniqueCount = Math.Min(system.GetClientActions().Count(), actionTypes.Length + 1);
         var keys = ContentKeyFunctions.GetHotbarBoundKeys();
 
-        Children.Clear();
-        for (var index = 0; index < count; index++)
+        for (var i = 0; i < uniqueCount; i++)
         {
-            Children.Add(MakeButton(index));
+            if (i >= ChildCount)
+            {
+                AddChild(MakeButton(i));
+            }
+
+            if (!actionTypes.TryGetValue(i, out var action))
+                action = null;
+            ((ActionButton) GetChild(i)).UpdateData(action, system);
+        }
+
+        for (var i = ChildCount - 1; i >= uniqueCount; i--)
+        {
+            RemoveChild(GetChild(i));
         }
 
         ActionButton MakeButton(int index)
@@ -55,20 +67,6 @@ public class ActionButtonContainer : GridContainer
         }
     }
 
-    public void SetActionData(ActionsSystem system, params EntityUid?[] actionTypes)
-    {
-        var uniqueCount = Math.Min(system.GetClientActions().Count(), actionTypes.Length + 1);
-        if (ChildCount != uniqueCount)
-            BuildActionButtons(uniqueCount);
-
-        for (var i = 0; i < uniqueCount; i++)
-        {
-            if (!actionTypes.TryGetValue(i, out var action))
-                action = null;
-            ((ActionButton) GetChild(i)).UpdateData(action, system);
-        }
-    }
-
     public void ClearActionData()
     {
         foreach (var button in Children)