Index: src/wp-includes/class-wp-image-editor.php
===================================================================
--- src/wp-includes/class-wp-image-editor.php	(revision 51606)
+++ src/wp-includes/class-wp-image-editor.php	(working copy)
@@ -591,13 +591,7 @@
 	 * @return string|false
 	 */
 	protected static function get_extension( $mime_type = null ) {
-		$extensions = explode( '|', array_search( $mime_type, wp_get_mime_types(), true ) );
-
-		if ( empty( $extensions[0] ) ) {
-			return false;
-		}
-
-		return $extensions[0];
+		return wp_default_extension_for_mime_type( $mime_type );
 	}
 }
 
Index: src/wp-includes/functions.php
===================================================================
--- src/wp-includes/functions.php	(revision 51606)
+++ src/wp-includes/functions.php	(working copy)
@@ -2595,6 +2595,9 @@
 				}
 			}
 		}
+
+		// If a different file type might be produced for an image, check filename uniqueness for that format.
+		$filename = _wp_check_alternate_output_format_uniqueness( $filename, $ext, $dir );
 	}
 
 	/**
@@ -2645,6 +2648,69 @@
 }
 
 /**
+ * Helper function for wp_unique_filename to check potential alternate output formats for images.
+ *
+ * @since 5.8.1
+ * @private
+ *
+ * @param string $filename
+ * @param string $ext
+ * @param string $dir
+ *
+ * @return string
+ */
+function _wp_check_alternate_output_format_uniqueness( $filename, $ext, $dir ) {
+	static $checking_alternates;
+
+	if ( empty( $checking_alternates ) ) {
+		$checking_alternates = true;
+		$file_type           = wp_check_filetype_and_ext( trailingslashit( $dir ) . $filename, $filename );
+		$mime_type           = $file_type['type'];
+
+		if ( ! empty( $mime_type ) && 0 === strpos( $mime_type, 'image/' ) ) {
+			$output_formats = apply_filters( 'image_editor_output_format', array(), trailingslashit( $dir ) . $filename, $mime_type );
+
+			if (
+				! empty( $output_formats ) &&
+				is_array( $output_formats ) &&
+				( ! empty( $output_formats[ $mime_type ] ) || in_array( $mime_type, $output_formats ) )
+			) {
+				$alt_mime_types = array();
+
+				foreach ( $output_formats as $source => $target ) {
+					if ( $source === $mime_type && $target === $mime_type ) {
+						continue;
+					} elseif ( $source === $mime_type ) {
+						$alt_mime_types[] = $target;
+					} elseif ( $target === $mime_type ) {
+						$alt_mime_types[] = $source;
+					}
+				}
+
+				$alt_mime_types = array_unique( $alt_mime_types );
+
+				foreach ( $alt_mime_types as $alt_mime_type ) {
+					$alt_ext = wp_default_extension_for_mime_type( $alt_mime_type );
+
+					if ( ! empty( $alt_ext ) && ".{$alt_ext}" !== $ext ) {
+						$alt_filename  = wp_basename( $filename, $ext ) . ".{$alt_ext}";
+						$alt_filename2 = wp_unique_filename( $dir, $alt_filename );
+
+						// If a potential clash was found for alternate format, use its unique filename.
+						if ( $alt_filename2 !== $alt_filename ) {
+							$filename = wp_basename( $alt_filename2, ".{$alt_ext}" ) . $ext;
+						}
+					}
+				}
+			}
+		}
+		$checking_alternates = false;
+	}
+
+	return $filename;
+}
+
+/**
  * Create a file in the upload folder with given content.
  *
  * If there is an error, then the key 'error' will exist with the error message.
@@ -3042,6 +3108,26 @@
 }
 
 /**
+ * Returns first matched extension from Mime-type,
+ * as mapped from wp_get_mime_types()
+ *
+ * @since 5.8.1
+ *
+ * @param string $mime_type
+ *
+ * @return string|false
+ */
+function wp_default_extension_for_mime_type( $mime_type = null ) {
+	$extensions = explode( '|', array_search( $mime_type, wp_get_mime_types(), true ) );
+
+	if ( empty( $extensions[0] ) ) {
+		return false;
+	}
+
+	return $extensions[0];
+}
+
+/**
  * Returns the real mime type of an image file.
  *
  * This depends on exif_imagetype() or getimagesize() to determine real mime types.
Index: tests/phpunit/tests/functions.php
===================================================================
--- tests/phpunit/tests/functions.php	(revision 51606)
+++ tests/phpunit/tests/functions.php	(working copy)
@@ -222,6 +222,52 @@
 	}
 
 	/**
+	 * @ticket 53668
+	 */
+	function test__wp_check_alternate_output_format_uniqueness() {
+		$testdir = DIR_TESTDATA . '/images/';
+
+		add_filter( 'upload_dir', array( $this, 'upload_dir_patch_basedir' ) );
+
+		// Standard test that wp_unique_filename allows usage if file does not exist yet.
+		$this->assertSame( 'abcdef.png', wp_unique_filename( $testdir, 'abcdef.png' ), 'non-existent file should not have name changed' );
+		// Difference in extension does not affect wp_unique_filename by default (canola.jpg exists).
+		$this->assertSame( 'canola.png', wp_unique_filename( $testdir, 'canola.png' ), 'clashing base filename but not extension should not have name changed' );
+		// Run again to prove no memory.
+		$this->assertSame( 'canola.png', wp_unique_filename( $testdir, 'canola.png' ), 'clashing base filename but not extension should not have name changed' );
+		// Actual clash recognized.
+		$this->assertSame( 'canola-1.jpg', wp_unique_filename( $testdir, 'canola.jpg' ), 'existing file should have name changed' );
+
+		// Now output jpg thumbnails for png files.
+		add_filter( 'image_editor_output_format', array( $this, 'image_editor_output_format_handler' ) );
+
+		// Standard test that wp_unique_filename allows usage if file does not exist yet.
+		$this->assertSame( 'abcdef.png', wp_unique_filename( $testdir, 'abcdef.png' ), 'non-existent file should not have name changed' );
+		// Difference in extension does affect wp_unique_filename when thumbnails use existing file's type.
+		$this->assertSame( 'canola-1.png', wp_unique_filename( $testdir, 'canola.png' ), 'clashing base filename but not extension should have name changed when thumbnails use existing file\'s type' );
+		// Run again to prove no memory.
+		$this->assertSame( 'canola-1.png', wp_unique_filename( $testdir, 'canola.png' ), 'clashing base filename but not extension should have name changed when thumbnails use existing file\'s type' );
+		// Actual clash recognized.
+		$this->assertSame( 'canola-1.jpg', wp_unique_filename( $testdir, 'canola.jpg' ), 'existing file should have name changed' );
+
+		remove_filter( 'image_editor_output_format', array( $this, 'image_editor_output_format_handler' ) );
+		remove_filter( 'upload_dir', array( $this, 'upload_dir_patch_basedir' ) );
+	}
+
+	/**
+	 * Changes the output format for a png file's generated thumbnails to be jpg.
+	 *
+	 * @param array $formats
+	 *
+	 * @return array
+	 */
+	public function image_editor_output_format_handler( $formats ) {
+		$formats['image/png'] = 'image/jpeg';
+
+		return $formats;
+	}
+
+	/**
 	 * @dataProvider data_is_not_serialized
 	 */
 	function test_maybe_serialize( $value ) {
@@ -1946,4 +1992,19 @@
 			array( 'application/activity+json, application/nojson', true ),
 		);
 	}
+
+	/**
+	 * @ticket 53668
+	 */
+	public function test_wp_default_extension_for_mime_type() {
+		$this->assertEquals( 'jpg', wp_default_extension_for_mime_type( 'image/jpeg' ), 'jpg not returned as default extension for "image/jpeg"' );
+		$this->assertNotEquals( 'jpeg', wp_default_extension_for_mime_type( 'image/jpeg' ), 'jpeg should not be returned as default extension for "image/jpeg"' );
+		$this->assertEquals( 'png', wp_default_extension_for_mime_type( 'image/png' ), 'png not returned as default extension for "image/png"' );
+		$this->assertFalse( wp_default_extension_for_mime_type( 'wibble/wobble' ), 'false not returned for unrecognized mime type' );
+		$this->assertFalse( wp_default_extension_for_mime_type(), 'false not returned when no mime type supplied' );
+		$this->assertFalse( wp_default_extension_for_mime_type( '' ), 'false not returned when empty string as mime type supplied' );
+		$this->assertFalse( wp_default_extension_for_mime_type( '   ' ), 'false not returned when empty string as mime type supplied' );
+		$this->assertFalse( wp_default_extension_for_mime_type( 123 ), 'false not returned when int as mime type supplied' );
+		$this->assertFalse( wp_default_extension_for_mime_type( null ), 'false not returned when null as mime type supplied' );
+	}
 }
