Index: src/wp-includes/js/media-views.js
===================================================================
--- src/wp-includes/js/media-views.js	(revision 29578)
+++ src/wp-includes/js/media-views.js	(working copy)
@@ -4897,8 +4897,12 @@
 				selection.single( model );
 				return;
 			} else if ( 'add' === method ) {
-				selection.add( model );
-				selection.single( model );
+				if ( this.selected() ) {
+					selection[ 'remove' ]( model );
+				} else {
+					selection.add( model );
+					selection.single( model );
+				}
 				return;
 			}