Index: src/wp-includes/class-wp-hook.php
===================================================================
--- src/wp-includes/class-wp-hook.php	(revision 40871)
+++ src/wp-includes/class-wp-hook.php	(working copy)
@@ -182,11 +182,9 @@
 		$exists = isset( $this->callbacks[ $priority ][ $function_key ] );
 		if ( $exists ) {
 			unset( $this->callbacks[ $priority ][ $function_key ] );
-			if ( ! $this->callbacks[ $priority ] ) {
-				unset( $this->callbacks[ $priority ] );
-				if ( $this->nesting_level > 0 ) {
-					$this->resort_active_iterations();
-				}
+
+			if ( $this->nesting_level > 0 ) {
+				$this->resort_active_iterations();
 			}
 		}
 		return $exists;
Index: tests/phpunit/tests/hooks/do_action.php
===================================================================
--- tests/phpunit/tests/hooks/do_action.php	(revision 40871)
+++ tests/phpunit/tests/hooks/do_action.php	(working copy)
@@ -169,4 +169,26 @@
 		$args = func_get_args();
 		$this->events[] = array('action' => __FUNCTION__, 'args'=>$args);
 	}
+
+	public function test_do_action_with_handler_that_removes_itself() {
+		$a = new MockAction();
+		$b = new MockAction();
+
+		$this->hook = new WP_Hook();
+
+		$this->hook->add_filter( 'handler_that_removes_itself', '__return_null', 10, 1 );
+		$this->hook->add_filter( 'handler_that_removes_itself', array( $this, '_handler_that_removes_itself' ), 20, 1 );
+		$this->hook->add_filter( 'handler_that_removes_itself', array( $a, 'action' ), 20 + 1, 1 );
+		$this->hook->add_filter( 'handler_that_removes_itself', array( $b, 'action' ), 20 + 2, 1 );
+
+		$this->hook->do_action( array( null ) );
+
+		$this->assertEquals( 1, $a->get_call_count(), 'One of the callbacks was never executed.' );
+		$this->assertEquals( 1, $b->get_call_count() );
+	}
+
+	public function _handler_that_removes_itself() {
+		$success = $this->hook->remove_filter( 'handler_that_removes_itself', array( $this, '_handler_that_removes_itself' ), 20 );
+		$this->assertTrue( $success );
+	}
 }
Index: tests/phpunit/tests/hooks/remove_filter.php
===================================================================
--- tests/phpunit/tests/hooks/remove_filter.php	(revision 40871)
+++ tests/phpunit/tests/hooks/remove_filter.php	(working copy)
@@ -17,7 +17,7 @@
 		$hook->add_filter( $tag, $callback, $priority, $accepted_args );
 		$hook->remove_filter( $tag, $callback, $priority );
 
-		$this->assertFalse( isset( $hook->callbacks[ $priority ] ) );
+		$this->assertEquals( 0, count( $hook->callbacks[ $priority ] ) );
 	}
 
 	public function test_remove_filter_with_object() {
@@ -31,7 +31,7 @@
 		$hook->add_filter( $tag, $callback, $priority, $accepted_args );
 		$hook->remove_filter( $tag, $callback, $priority );
 
-		$this->assertFalse( isset( $hook->callbacks[ $priority ] ) );
+		$this->assertEquals( 0, count( $hook->callbacks[ $priority ] ) );
 	}
 
 	public function test_remove_filter_with_static_method() {
@@ -44,7 +44,7 @@
 		$hook->add_filter( $tag, $callback, $priority, $accepted_args );
 		$hook->remove_filter( $tag, $callback, $priority );
 
-		$this->assertFalse( isset( $hook->callbacks[ $priority ] ) );
+		$this->assertEquals( 0, count( $hook->callbacks[ $priority ] ) );
 	}
 
 	public function test_remove_filters_with_another_at_same_priority() {
@@ -75,7 +75,7 @@
 		$hook->add_filter( $tag, $callback_two, $priority + 1, $accepted_args );
 
 		$hook->remove_filter( $tag, $callback_one, $priority );
-		$this->assertFalse( isset( $hook->callbacks[ $priority ] ) );
+		$this->assertEquals( 0, count( $hook->callbacks[ $priority ] ) );
 		$this->assertCount( 1, $hook->callbacks[ $priority + 1 ] );
 	}
 }