Index: src/wp-admin/js/customize-controls.js
===================================================================
--- src/wp-admin/js/customize-controls.js	(revision 37080)
+++ src/wp-admin/js/customize-controls.js	(working copy)
@@ -91,7 +91,7 @@
 			}
 
 			// Note that we can't use :focusable due to a jQuery UI issue. See: https://github.com/jquery/jquery-ui/pull/1583
-			focusContainer.find( 'input, select, textarea, button, object, a[href], [tabindex]' ).filter( ':visible' ).first().focus();
+			focusContainer.find( 'input, select, textarea, button, object, a[href], [tabindex]' ).filter( ':visible' ).not( '.remove-button' ).first().focus();
 		};
 		if ( params.completeCallback ) {
 			completeCallback = params.completeCallback;