Index: src/wp-includes/js/wplink.js
===================================================================
--- src/wp-includes/js/wplink.js	(revision 33589)
+++ src/wp-includes/js/wplink.js	(working copy)
@@ -365,8 +365,11 @@
 			}
 
 			link = getLink();
-			text = inputs.text.val() || attrs.href;
 
+			if ( inputs.wrap.hasClass( 'has-text-field' ) ) {
+				text = inputs.text.val() || attrs.href;
+			}
+
 			if ( link ) {
 				if ( text ) {
 					if ( 'innerText' in link ) {