Index: src/wp-includes/class-wp-image-editor.php
===================================================================
--- src/wp-includes/class-wp-image-editor.php	(revision 51607)
+++ src/wp-includes/class-wp-image-editor.php	(working copy)
@@ -591,13 +591,11 @@
 	 * @return string|false
 	 */
 	protected static function get_extension( $mime_type = null ) {
-		$extensions = explode( '|', array_search( $mime_type, wp_get_mime_types(), true ) );
-
-		if ( empty( $extensions[0] ) ) {
+		if ( empty( $mime_type ) ) {
 			return false;
 		}
 
-		return $extensions[0];
+		return wp_get_default_extension_for_mime_type( $mime_type );
 	}
 }
 
Index: src/wp-includes/functions.php
===================================================================
--- src/wp-includes/functions.php	(revision 51607)
+++ src/wp-includes/functions.php	(working copy)
@@ -2486,7 +2486,6 @@
 function wp_unique_filename( $dir, $filename, $unique_filename_callback = null ) {
 	// Sanitize the file name before we begin processing.
 	$filename = sanitize_file_name( $filename );
-	$ext2     = null;
 
 	// Separate the filename into a name and extension.
 	$ext  = pathinfo( $filename, PATHINFO_EXTENSION );
@@ -2501,50 +2500,44 @@
 		$name = '';
 	}
 
+	// Reconstruct sanitized filename with lower case extension.
+	$orig_ext = $ext;
+	$ext      = strtolower( $ext );
+	$filename = pathinfo( $filename, PATHINFO_FILENAME ) . $ext;
+
 	/*
 	 * Increment the file number until we have a unique file to save in $dir.
 	 * Use callback if supplied.
 	 */
 	if ( $unique_filename_callback && is_callable( $unique_filename_callback ) ) {
-		$filename = call_user_func( $unique_filename_callback, $dir, $name, $ext );
+		$filename = call_user_func( $unique_filename_callback, $dir, $name, $orig_ext );
 	} else {
 		$number = '';
 		$fname  = pathinfo( $filename, PATHINFO_FILENAME );
 
+		// If filename already versioned, get version and un-versioned filename.
+		if ( preg_match( '/-(\d)$/', $fname, $matches ) ) {
+			$fname  = preg_replace( '/' . $matches[0] . '$/', '', $fname );
+			$number = (int) $matches[1];
+		}
+
 		// Always append a number to file names that can potentially match image sub-size file names.
 		if ( $fname && preg_match( '/-(?:\d+x\d+|scaled|rotated)$/', $fname ) ) {
-			$number = 1;
+			$number = (int) $number + 1;
 
 			// At this point the file name may not be unique. This is tested below and the $number is incremented.
 			$filename = str_replace( "{$fname}{$ext}", "{$fname}-{$number}{$ext}", $filename );
 		}
 
-		// Change '.ext' to lower case.
-		if ( $ext && strtolower( $ext ) != $ext ) {
-			$ext2      = strtolower( $ext );
-			$filename2 = preg_replace( '|' . preg_quote( $ext ) . '$|', $ext2, $filename );
+		// Check for both lower and upper case extension or image sub-sizes may be overwritten.
+		$uc_ext          = strtoupper( $ext );
+		$uc_ext_filename = preg_replace( '|' . preg_quote( $ext ) . '$|', $uc_ext, $filename );
 
-			// Check for both lower and upper case extension or image sub-sizes may be overwritten.
-			while ( file_exists( $dir . "/{$filename}" ) || file_exists( $dir . "/{$filename2}" ) ) {
-				$new_number = (int) $number + 1;
-				$filename   = str_replace( array( "-{$number}{$ext}", "{$number}{$ext}" ), "-{$new_number}{$ext}", $filename );
-				$filename2  = str_replace( array( "-{$number}{$ext2}", "{$number}{$ext2}" ), "-{$new_number}{$ext2}", $filename2 );
-				$number     = $new_number;
-			}
-
-			$filename = $filename2;
-		} else {
-			while ( file_exists( $dir . "/{$filename}" ) ) {
-				$new_number = (int) $number + 1;
-
-				if ( '' === "{$number}{$ext}" ) {
-					$filename = "{$filename}-{$new_number}";
-				} else {
-					$filename = str_replace( array( "-{$number}{$ext}", "{$number}{$ext}" ), "-{$new_number}{$ext}", $filename );
-				}
-
-				$number = $new_number;
-			}
+		while ( file_exists( $dir . "/{$filename}" ) || file_exists( $dir . "/{$uc_ext_filename}" ) ) {
+			$new_number      = (int) $number + 1;
+			$filename        = str_replace( array( "{$fname}-{$number}{$ext}", "{$fname}{$number}{$ext}" ), "{$fname}-{$new_number}{$ext}", $filename );
+			$uc_ext_filename = str_replace( array( "{$fname}-{$number}{$uc_ext}", "{$fname}{$number}{$uc_ext}" ), "{$fname}-{$new_number}{$uc_ext}", $uc_ext_filename );
+			$number          = $new_number;
 		}
 
 		// Prevent collisions with existing file names that contain dimension-like strings
@@ -2579,9 +2572,6 @@
 			}
 
 			if ( ! empty( $files ) ) {
-				// The extension case may have changed above.
-				$new_ext = ! empty( $ext2 ) ? $ext2 : $ext;
-
 				// Ensure this never goes into infinite loop
 				// as it uses pathinfo() and regex in the check, but string replacement for the changes.
 				$count = count( $files );
@@ -2589,12 +2579,15 @@
 
 				while ( $i <= $count && _wp_check_existing_file_names( $filename, $files ) ) {
 					$new_number = (int) $number + 1;
-					$filename   = str_replace( array( "-{$number}{$new_ext}", "{$number}{$new_ext}" ), "-{$new_number}{$new_ext}", $filename );
+					$filename   = str_replace( array( "{$fname}-{$number}{$ext}", "{$fname}{$number}{$ext}" ), "{$fname}-{$new_number}{$ext}", $filename );
 					$number     = $new_number;
 					$i++;
 				}
 			}
 		}
+
+		// If a different file type might be produced for an image, check filename uniqueness for that format.
+		$filename = _wp_check_alternate_output_format_uniqueness( $filename, $ext, $dir );
 	}
 
 	/**
@@ -2603,11 +2596,11 @@
 	 * @since 4.5.0
 	 *
 	 * @param string        $filename                 Unique file name.
-	 * @param string        $ext                      File extension, eg. ".png".
+	 * @param string        $orig_ext                 File extension using original case, e.g. ".png" or ".PNG".
 	 * @param string        $dir                      Directory path.
 	 * @param callable|null $unique_filename_callback Callback function that generates the unique file name.
 	 */
-	return apply_filters( 'wp_unique_filename', $filename, $ext, $dir, $unique_filename_callback );
+	return apply_filters( 'wp_unique_filename', $filename, $orig_ext, $dir, $unique_filename_callback );
 }
 
 /**
@@ -2645,6 +2638,77 @@
 }
 
 /**
+ * Helper function for wp_unique_filename to check potential alternate output formats for images.
+ *
+ * @since 5.8.1
+ * @private
+ *
+ * @param string $filename
+ * @param string $ext
+ * @param string $dir
+ *
+ * @return string
+ */
+function _wp_check_alternate_output_format_uniqueness( $filename, $ext, $dir ) {
+	static $checking_alternates;
+
+	if ( empty( $checking_alternates ) ) {
+		$checking_alternates = true;
+		$filename_changed    = false;
+		$file_type           = wp_check_filetype_and_ext( trailingslashit( $dir ) . $filename, $filename );
+		$mime_type           = $file_type['type'];
+
+		if ( ! empty( $mime_type ) && 0 === strpos( $mime_type, 'image/' ) ) {
+			$output_formats = apply_filters( 'image_editor_output_format', array(), trailingslashit( $dir ) . $filename, $mime_type );
+
+			if ( ! empty( $output_formats ) && is_array( $output_formats ) ) {
+				// Temporarily add uploaded type to alts for simpler analysis.
+				$alt_mime_types = array( $mime_type );
+
+				// Alternate thumbnail format for uploaded file?
+				if ( ! empty( $output_formats[ $mime_type ] ) ) {
+					$alt_mime_types[] = $output_formats[ $mime_type ];
+				}
+
+				// Any other formats using uploaded or alternate format for thumbnails?
+				$alt_mime_types = array_merge( $alt_mime_types, array_keys( array_intersect( $output_formats, $alt_mime_types ) ) );
+
+				// Remove uploaded mime type as we've already tested that.
+				$alt_mime_types = array_diff( $alt_mime_types, array( $mime_type ) );
+			}
+
+			if ( ! empty( $alt_mime_types ) ) {
+				$alt_mime_types = array_unique( $alt_mime_types );
+
+				foreach ( $alt_mime_types as $alt_mime_type ) {
+					$alt_ext = wp_get_default_extension_for_mime_type( $alt_mime_type );
+
+					if ( ! empty( $alt_ext ) && ".{$alt_ext}" !== $ext ) {
+						$alt_filename  = wp_basename( $filename, $ext ) . ".{$alt_ext}";
+						$alt_filename2 = wp_unique_filename( $dir, $alt_filename );
+
+						// If a potential clash was found for alternate format, use its unique filename.
+						if ( $alt_filename2 !== $alt_filename ) {
+							$filename         = wp_basename( $alt_filename2, ".{$alt_ext}" ) . $ext;
+							$filename_changed = true;
+						}
+					}
+				}
+			}
+		}
+		$checking_alternates = false;
+
+		// Double check that incremented filename for original type or first tested alternate types do not clash.
+		if ( $filename_changed ) {
+
+			return wp_unique_filename( $dir, $filename );
+		}
+	}
+
+	return $filename;
+}
+
+/**
  * Create a file in the upload folder with given content.
  *
  * If there is an error, then the key 'error' will exist with the error message.
@@ -3042,6 +3106,27 @@
 }
 
 /**
+ * Returns first matched extension from Mime-type,
+ * as mapped from wp_get_mime_types()
+ *
+ * @since 5.8.1
+ *
+ * @param string $mime_type
+ *
+ * @return string|false
+ *
+ */
+function wp_get_default_extension_for_mime_type( $mime_type ) {
+	$extensions = explode( '|', array_search( $mime_type, wp_get_mime_types(), true ) );
+
+	if ( empty( $extensions[0] ) ) {
+		return false;
+	}
+
+	return $extensions[0];
+}
+
+/**
  * Returns the real mime type of an image file.
  *
  * This depends on exif_imagetype() or getimagesize() to determine real mime types.
Index: tests/phpunit/tests/functions.php
===================================================================
--- tests/phpunit/tests/functions.php	(revision 51607)
+++ tests/phpunit/tests/functions.php	(working copy)
@@ -222,6 +222,91 @@
 	}
 
 	/**
+	 * @ticket 53668
+	 */
+	function test__wp_check_alternate_output_format_uniqueness() {
+		$testdir = DIR_TESTDATA . '/images/';
+
+		add_filter( 'upload_dir', array( $this, 'upload_dir_patch_basedir' ) );
+
+		// Standard test that wp_unique_filename allows usage if file does not exist yet.
+		$this->assertSame( 'abcdef.png', wp_unique_filename( $testdir, 'abcdef.png' ), 'The abcdef.png image does not exist. The name does not need to be made unique.' );
+		// Difference in extension does not affect wp_unique_filename by default (canola.jpg exists).
+		$this->assertSame( 'canola.png', wp_unique_filename( $testdir, 'canola.png' ), 'The canola.jpg image exists. Clashing base filename but not extension should not have name changed.' );
+		// Run again with upper case extension.
+		$this->assertSame( 'canola.png', wp_unique_filename( $testdir, 'canola.PNG' ), 'The canola.jpg image exists. Clashing base filename but not extension should not have name changed.' );
+		// Actual clash recognized.
+		$this->assertSame( 'canola-1.jpg', wp_unique_filename( $testdir, 'canola.jpg' ), 'The canola.jpg image exists. Uploading canola.jpg again should have unique name.' );
+		// Future clash by regenerated thumbnails not applicable.
+		$this->assertSame( 'codeispoetry.jpg', wp_unique_filename( $testdir, 'codeispoetry.jpg' ), 'The codeispoetry.png image exists. Uploading codeispoetry.jpg does not need unique name.' );
+
+		// When creating sub-sizes convert uploaded PNG images to JPG.
+		add_filter( 'image_editor_output_format', array( $this, 'image_editor_output_format_handler' ) );
+
+		// Standard test that wp_unique_filename allows usage if file does not exist yet.
+		$this->assertSame( 'abcdef.png', wp_unique_filename( $testdir, 'abcdef.png' ), 'The abcdef.png image does not exist. Its name should not be changed.' );
+		// Standard test that wp_unique_filename allows usage if file does not exist yet.
+		$this->assertSame( 'abcdef.bmp', wp_unique_filename( $testdir, 'abcdef.bmp' ), 'The abcdef.bmp and abcdef.pct images do not exist. When uploading abcdef.bmp its name should not be changed.' );
+		// Difference in extension does affect wp_unique_filename when thumbnails use existing file's type.
+		$this->assertSame( 'canola-1.png', wp_unique_filename( $testdir, 'canola.png' ), 'The canola.jpg image exists. Uploading canola.png that will be converted to canola.jpg should produce unique file name.' );
+		// Run again with upper case extension.
+		$this->assertSame( 'canola-1.png', wp_unique_filename( $testdir, 'canola.PNG' ), 'The canola.jpg image exists. Uploading canola.PNG that will be converted to canola.jpg should produce unique file name.' );
+		// Actual clash recognized.
+		$this->assertSame( 'canola-1.jpg', wp_unique_filename( $testdir, 'canola.jpg' ), 'Existing file should have name changed.' );
+		// Actual clash with images with different extensions.
+		$this->assertSame( 'test-image-3.png', wp_unique_filename( $testdir, 'test-image.png' ), 'The test-image.png, test-image-1-100x100.jpg, and test-image-2.gif images exist. All of them may be intersected when creating sub-sizes for the new image: test-image.png, so its filename should be unique.' );
+		// Future clash by regenerated thumbnails recognized.
+		$this->assertSame( 'codeispoetry-1.jpg', wp_unique_filename( $testdir, 'codeispoetry.jpg' ), 'The codeispoetry.png image exists. When regenerating thumbnails for it they will be converted to JPG. The name of the newly uploaded codeispoetry.jpg should be made unique.' );
+
+		// Ensure plugins take part in analysis of alternate extensions.
+		add_filter( 'wp_unique_filename', array( $this, 'wp_unique_filename_handler' ), 10, 4 );
+
+		// Actual clash with images with different extensions.
+		$this->assertSame( 'test-image-4.png', wp_unique_filename( $testdir, 'test-image.png' ), 'The test-image.png, test-image-1-100x100.jpg, test-image-2.gif and offloaded test-image-3.jpg images exist. All of them may be intersected when creating sub-sizes for the new image: test-image.png, so its filename should be unique.' );
+
+		remove_filter( 'wp_unique_filename', array( $this, 'wp_unique_filename_handler' ) );
+		remove_filter( 'image_editor_output_format', array( $this, 'image_editor_output_format_handler' ) );
+		remove_filter( 'upload_dir', array( $this, 'upload_dir_patch_basedir' ) );
+	}
+
+	/**
+	 * Changes the output format when editing images. When uploading a PNG file
+	 * it will be converted to JPG (if the image editor in PHP supports it).
+	 *
+	 * @param array $formats
+	 *
+	 * @return array
+	 */
+	public function image_editor_output_format_handler( $formats ) {
+		$formats['image/png'] = 'image/jpeg';
+		$formats['image/gif'] = 'image/jpeg';
+		$formats['image/pct'] = 'image/bmp';
+
+		return $formats;
+	}
+
+	/**
+	 * Pretend that a filename has been uploaded but removed from the server's filesystem
+	 * e.g. by an offloader plugin on a cluster with ephemeral filesystem.
+	 *
+	 * @param string   $filename
+	 * @param string   $orig_ext
+	 * @param string   $dir
+	 * @param callable $unique_filename_callback
+	 *
+	 * @return string
+	 */
+	public function wp_unique_filename_handler( $filename, $orig_ext, $dir, $unique_filename_callback ) {
+		if ( 'test-image-3.jpg' === $filename ) {
+			// Bump the filename version and re-check.
+			$filename = 'test-image-4.jpg';
+			$filename = wp_unique_filename( $dir, $filename );
+		}
+
+		return $filename;
+	}
+
+	/**
 	 * @dataProvider data_is_not_serialized
 	 */
 	function test_maybe_serialize( $value ) {
@@ -1946,4 +2031,18 @@
 			array( 'application/activity+json, application/nojson', true ),
 		);
 	}
+
+	/**
+	 * @ticket 53668
+	 */
+	public function test_wp_get_default_extension_for_mime_type() {
+		$this->assertEquals( 'jpg', wp_get_default_extension_for_mime_type( 'image/jpeg' ), 'jpg not returned as default extension for "image/jpeg"' );
+		$this->assertNotEquals( 'jpeg', wp_get_default_extension_for_mime_type( 'image/jpeg' ), 'jpeg should not be returned as default extension for "image/jpeg"' );
+		$this->assertEquals( 'png', wp_get_default_extension_for_mime_type( 'image/png' ), 'png not returned as default extension for "image/png"' );
+		$this->assertFalse( wp_get_default_extension_for_mime_type( 'wibble/wobble' ), 'false not returned for unrecognized mime type' );
+		$this->assertFalse( wp_get_default_extension_for_mime_type( '' ), 'false not returned when empty string as mime type supplied' );
+		$this->assertFalse( wp_get_default_extension_for_mime_type( '   ' ), 'false not returned when empty string as mime type supplied' );
+		$this->assertFalse( wp_get_default_extension_for_mime_type( 123 ), 'false not returned when int as mime type supplied' );
+		$this->assertFalse( wp_get_default_extension_for_mime_type( null ), 'false not returned when null as mime type supplied' );
+	}
 }
