Index: wp-admin/includes/file.php
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- wp-admin/includes/file.php	(revision 964d0784befec3f3e01bf9844a39f2b1362e995f)
+++ wp-admin/includes/file.php	(date 1600672597504)
@@ -928,7 +928,7 @@
 	$url = $uploads['url'] . "/$filename";
 
 	if ( is_multisite() ) {
-		delete_transient( 'dirsize_cache' );
+		clear_dirsize_cache( $new_file );
 	}
 
 	/**
Index: wp-includes/post.php
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- wp-includes/post.php	(revision 964d0784befec3f3e01bf9844a39f2b1362e995f)
+++ wp-includes/post.php	(date 1600673077529)
@@ -5827,7 +5827,7 @@
 	$file         = get_attached_file( $post_id );
 
 	if ( is_multisite() ) {
-		delete_transient( 'dirsize_cache' );
+		clear_dirsize_cache( $file );
 	}
 
 	/**
Index: wp-includes/functions.php
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- wp-includes/functions.php	(revision 964d0784befec3f3e01bf9844a39f2b1362e995f)
+++ wp-includes/functions.php	(date 1600673419342)
@@ -7500,26 +7500,16 @@
  * @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout.
  */
 function get_dirsize( $directory, $max_execution_time = null ) {
-	$dirsize = get_transient( 'dirsize_cache' );
-
-	if ( is_array( $dirsize ) && isset( $dirsize[ $directory ]['size'] ) ) {
-		return $dirsize[ $directory ]['size'];
-	}
-
-	if ( ! is_array( $dirsize ) ) {
-		$dirsize = array();
-	}
 
 	// Exclude individual site directories from the total when checking the main site of a network,
 	// as they are subdirectories and should not be counted.
 	if ( is_multisite() && is_main_site() ) {
-		$dirsize[ $directory ]['size'] = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time );
+		$size = recurse_dirsize( $directory, $directory . '/sites', $max_execution_time );
 	} else {
-		$dirsize[ $directory ]['size'] = recurse_dirsize( $directory, null, $max_execution_time );
+		$size = recurse_dirsize( $directory, null, $max_execution_time );
 	}
 
-	set_transient( 'dirsize_cache', $dirsize, HOUR_IN_SECONDS );
-	return $dirsize[ $directory ]['size'];
+	return $size;
 }
 
 /**
@@ -7531,18 +7521,32 @@
  * @since MU (3.0.0)
  * @since 4.3.0 $exclude parameter added.
  * @since 5.2.0 $max_execution_time parameter added.
+ * @since 5.6.0 $directory_cache parameter added.
  *
  * @param string       $directory          Full path of a directory.
  * @param string|array $exclude            Optional. Full path of a subdirectory to exclude from the total,
  *                                         or array of paths. Expected without trailing slash(es).
  * @param int          $max_execution_time Maximum time to run before giving up. In seconds. The timeout is global
  *                                         and is measured from the moment WordPress started to load.
+ * @param array        $directory_cache    Optional. Array of cached directory paths.
+ *
  * @return int|false|null Size in bytes if a valid directory. False if not. Null if timeout.
  */
-function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null ) {
+function recurse_dirsize( $directory, $exclude = null, $max_execution_time = null, &$directory_cache = null ) {
 	$size = 0;
 
 	$directory = untrailingslashit( $directory );
+	$cache_path = normalize_dirsize_cache_path( $directory );
+	$save_cache = false;
+
+	if ( ! isset( $directory_cache ) ) {
+		$directory_cache = get_transient( 'dirsize_cache' );
+		$save_cache      = true;
+	}
+
+	if ( isset( $directory_cache[ $cache_path ] ) ) {
+		return $directory_cache[ $cache_path ];
+	}
 
 	if ( ! file_exists( $directory ) || ! is_dir( $directory ) || ! is_readable( $directory ) ) {
 		return false;
@@ -7578,7 +7582,7 @@
 				if ( is_file( $path ) ) {
 					$size += filesize( $path );
 				} elseif ( is_dir( $path ) ) {
-					$handlesize = recurse_dirsize( $path, $exclude, $max_execution_time );
+					$handlesize = recurse_dirsize( $path, $exclude, $max_execution_time, $directory_cache );
 					if ( $handlesize > 0 ) {
 						$size += $handlesize;
 					}
@@ -7593,9 +7597,62 @@
 		}
 		closedir( $handle );
 	}
+
+	$directory_cache[ $cache_path ] = $size;
+
+	// Only write the transient on the top level call and not on recursive calls
+	if ( $save_cache ) {
+		set_transient( 'dirsize_cache', $directory_cache );
+	}
+
 	return $size;
 }
 
+/**
+ * Clear dirsize_cache
+ *
+ * Remove the current directory and all parent directories
+ * from the dirsize_cache transient.
+ *
+ * @since 5.6.0
+ *
+ * @param string $path Full path of a directory.
+ */
+function clear_dirsize_cache( $path ) {
+	$directory_cache = get_transient( 'dirsize_cache' );
+
+	if ( empty( $directory_cache ) ) {
+		return;
+	}
+
+	$cache_path = normalize_dirsize_cache_path( $path );
+	unset( $directory_cache[ $cache_path ] );
+
+	while ( DIRECTORY_SEPARATOR !== $cache_path && '.' !== $cache_path ) {
+		$cache_path = dirname( $cache_path );
+		unset( $directory_cache[ $cache_path ] );
+	}
+
+	set_transient( 'dirsize_cache', $directory_cache );
+}
+
+/**
+ * Normalize dirsize cache path.
+ *
+ * Ensures array keys follow the same format.
+ *
+ * @param string $path
+ *
+ * @since 5.6.0
+ *
+ * @return string
+ */
+function normalize_dirsize_cache_path( $path ) {
+	$path = str_replace( ABSPATH, '', $path );
+
+	return untrailingslashit( $path );
+}
+
 /**
  * Checks compatibility with the current WordPress version.
  *
