diff --git src/wp-includes/class-wp-term-query.php src/wp-includes/class-wp-term-query.php
index 12e7dec242..a93e67b910 100644
--- src/wp-includes/class-wp-term-query.php
+++ src/wp-includes/class-wp-term-query.php
@@ -551,6 +551,12 @@ class WP_Term_Query {
 			$limits = '';
 		}
 
+		$do_distinct = false;
+
+		// When querying for a specific number of terms, let MySQL eliminate duplicates.
+		if ( ! empty( $limits ) && 'all_with_object_id' !== $args['fields'] ) {
+			$do_distinct = true;
+		}
 
 		if ( ! empty( $args['search'] ) ) {
 			$this->sql_clauses['where']['search'] = $this->get_search_sql( $args['search'] );
@@ -568,8 +574,7 @@ class WP_Term_Query {
 		if ( ! empty( $meta_clauses ) ) {
 			$join .= $mq_sql['join'];
 			$this->sql_clauses['where']['meta_query'] = preg_replace( '/^\s*AND\s*/', '', $mq_sql['where'] );
-			$distinct .= "DISTINCT";
-
+			$do_distinct = true;
 		}
 
 		$selects = array();
@@ -631,6 +636,8 @@ class WP_Term_Query {
 
 		$where = implode( ' AND ', $this->sql_clauses['where'] );
 
+		$distinct = $do_distinct ? 'DISTINCT' : '';
+
 		/**
 		 * Filters the terms query SQL clauses.
 		 *
diff --git tests/phpunit/tests/term/query.php tests/phpunit/tests/term/query.php
index b2e2a27621..73a4a749cd 100644
--- tests/phpunit/tests/term/query.php
+++ tests/phpunit/tests/term/query.php
@@ -427,4 +427,53 @@ class Tests_Term_Query extends WP_UnitTestCase {
 		$terms = wp_get_object_terms( $post_id, array( 'category', 'wptests_tax' ) );
 		$this->assertEquals( array( $term_ids[1], $term_ids[0], 1 ), wp_list_pluck( $terms, 'term_id' ) );
 	}
+
+	/**
+	 * @ticket 41796
+	 */
+	public function test_number_should_work_with_object_ids() {
+		register_taxonomy( 'wptests_tax', 'post' );
+
+		$term_1 = self::factory()->term->create( array( 'taxonomy' => 'wptests_tax' ) );
+		$term_2 = self::factory()->term->create( array( 'taxonomy' => 'wptests_tax' ) );
+
+		$post_1 = self::factory()->post->create();
+		$post_2 = self::factory()->post->create();
+
+		wp_set_object_terms( $post_1, array( $term_1, $term_2 ), 'wptests_tax' );
+		wp_set_object_terms( $post_2, array( $term_1 ), 'wptests_tax' );
+
+		$q = new WP_Term_Query( array(
+			'taxonomy' => 'wptests_tax',
+			'object_ids' => array( $post_1, $post_2 ),
+			'number' => 2,
+		) );
+
+		$this->assertEqualSets( array( $term_1, $term_2 ), wp_list_pluck( $q->terms, 'term_id' ) );
+	}
+
+	/**
+	 * @ticket 41796
+	 */
+	public function test_number_should_work_with_object_ids_and_all_with_object_id() {
+		register_taxonomy( 'wptests_tax', 'post' );
+
+		$term_1 = self::factory()->term->create( array( 'taxonomy' => 'wptests_tax' ) );
+		$term_2 = self::factory()->term->create( array( 'taxonomy' => 'wptests_tax' ) );
+
+		$post_1 = self::factory()->post->create();
+		$post_2 = self::factory()->post->create();
+
+		wp_set_object_terms( $post_1, array( $term_1, $term_2 ), 'wptests_tax' );
+		wp_set_object_terms( $post_2, array( $term_1 ), 'wptests_tax' );
+
+		$q = new WP_Term_Query( array(
+			'taxonomy' => 'wptests_tax',
+			'object_ids' => array( $post_1, $post_2 ),
+			'fields' => 'all_with_object_id',
+			'number' => 2,
+		) );
+
+		$this->assertEqualSets( array( $term_1, $term_1 ), wp_list_pluck( $q->terms, 'term_id' ) );
+	}
 }