Changeset 58076
- Timestamp:
- 05/02/2024 01:57:49 PM (9 months ago)
- Location:
- trunk
- Files:
-
- 1 added
- 6 deleted
- 9 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/.github/workflows/performance.yml
r57918 r58076 67 67 # - Install WordPress Importer plugin. 68 68 # - Import mock data. 69 # - Deactivate WordPress Importer plugin. 69 70 # - Update permalink structure. 71 # - Install additional languages. 72 # - Disable external HTTP requests. 73 # - Disable cron. 74 # - List defined constants. 70 75 # - Install MU plugin. 71 76 # - Run performance tests (current commit). 72 # - Print performance tests results. 73 # - Check out target commit (target branch or previous commit). 74 # - Switch Node.js versions if necessary. 75 # - Install npm dependencies. 76 # - Build WordPress. 77 # - Download previous build artifact (target branch or previous commit). 78 # - Download artifact. 79 # - Unzip the build. 77 80 # - Run any database upgrades. 81 # - Flush cache. 82 # - Delete expired transients. 78 83 # - Run performance tests (previous/target commit). 79 # - Print target performance tests results.80 # - Reset to original commit.81 # - Switch Node.js versions if necessary.82 # - Install npm dependencies.83 84 # - Set the environment to the baseline version. 84 85 # - Run any database upgrades. 86 # - Flush cache. 87 # - Delete expired transients. 85 88 # - Run baseline performance tests. 86 # - Print baseline performance tests results.87 # - Compare results with base.89 # - Archive artifacts. 90 # - Compare results. 88 91 # - Add workflow summary. 89 92 # - Set the base sha. … … 91 94 # - Publish performance results. 92 95 # - Ensure version-controlled files are not modified or deleted. 93 # - Dispatch workflow run.94 96 performance: 95 name: Run performance tests 97 name: Run performance tests ${{ matrix.memcached && '(with memcached)' || '' }} 96 98 runs-on: ubuntu-latest 97 99 permissions: 98 100 contents: read 99 101 if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) && ! contains( github.event.before, '00000000' ) }} 100 102 strategy: 103 fail-fast: false 104 matrix: 105 memcached: [ true, false ] 106 env: 107 LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }} 101 108 steps: 102 109 - name: Configure environment variables … … 128 135 129 136 - name: Install Playwright browsers 130 run: npx playwright install --with-deps 137 run: npx playwright install --with-deps chromium 131 138 132 139 - name: Build WordPress … … 134 141 135 142 - name: Start Docker environment 136 run: | 137 npm run env:start 143 run: npm run env:start 144 145 - name: Install object cache drop-in 146 if: ${{ matrix.memcached }} 147 run: cp src/wp-content/object-cache.php build/wp-content/object-cache.php 138 148 139 149 - name: Log running Docker containers … … 161 171 rm themeunittestdata.wordpress.xml 162 172 173 - name: Deactivate WordPress Importer plugin 174 run: npm run env:cli -- plugin deactivate wordpress-importer --path=/var/www/${{ env.LOCAL_DIR }} 175 163 176 - name: Update permalink structure 164 run: | 165 npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }} 177 run: npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }} 166 178 167 179 - name: Install additional languages … … 171 183 npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }} 172 184 185 # Prevent background update checks from impacting test stability. 186 - name: Disable external HTTP requests 187 run: npm run env:cli -- config set WP_HTTP_BLOCK_EXTERNAL true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }} 188 189 # Prevent background tasks from impacting test stability. 190 - name: Disable cron 191 run: npm run env:cli -- config set DISABLE_WP_CRON true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }} 192 193 - name: List defined constants 194 run: npm run env:cli -- config list --path=/var/www/${{ env.LOCAL_DIR }} 195 173 196 - name: Install MU plugin 174 197 run: | … … 179 202 run: npm run test:performance 180 203 181 - name: Print performance tests results 182 run: node ./tests/performance/results.js 183 184 - name: Check out target commit (target branch or previous commit) 185 run: | 186 if [[ -z "$TARGET_REF" ]]; then 187 git fetch -n origin $TARGET_SHA 188 else 189 git fetch -n origin $TARGET_REF 190 fi 191 git reset --hard $TARGET_SHA 192 193 - name: Set up Node.js 194 uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 195 with: 196 node-version-file: '.nvmrc' 197 cache: npm 198 199 - name: Install npm dependencies 200 run: npm ci 201 202 - name: Build WordPress 203 run: npm run build 204 - name: Download previous build artifact (target branch or previous commit) 205 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 206 id: get-previous-build 207 with: 208 script: | 209 const artifacts = await github.rest.actions.listArtifactsForRepo({ 210 owner: context.repo.owner, 211 repo: context.repo.repo, 212 name: 'wordpress-build-' + process.env.TARGET_SHA, 213 }); 214 215 const matchArtifact = artifacts.data.artifacts[0]; 216 217 if ( ! matchArtifact ) { 218 core.setFailed( 'No artifact found!' ); 219 return false; 220 } 221 222 const download = await github.rest.actions.downloadArtifact( { 223 owner: context.repo.owner, 224 repo: context.repo.repo, 225 artifact_id: matchArtifact.id, 226 archive_format: 'zip', 227 } ); 228 229 const fs = require( 'fs' ); 230 fs.writeFileSync( '${{ github.workspace }}/before.zip', Buffer.from( download.data ) ) 231 232 return true; 233 234 - name: Unzip the build 235 if: ${{ steps.get-previous-build.outputs.result }} 236 run: | 237 unzip ${{ github.workspace }}/before.zip 238 unzip -o ${{ github.workspace }}/wordpress.zip 204 239 205 240 - name: Run any database upgrades 241 if: ${{ steps.get-previous-build.outputs.result }} 206 242 run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }} 207 243 208 - name: Run target performance tests (base/previous commit) 244 - name: Flush cache 245 if: ${{ steps.get-previous-build.outputs.result }} 246 run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }} 247 248 - name: Delete expired transients 249 if: ${{ steps.get-previous-build.outputs.result }} 250 run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }} 251 252 - name: Run target performance tests (previous/target commit) 253 if: ${{ steps.get-previous-build.outputs.result }} 209 254 env: 210 255 TEST_RESULTS_PREFIX: before 211 256 run: npm run test:performance 212 257 213 - name: Print target performance tests results214 env:215 TEST_RESULTS_PREFIX: before216 run: node ./tests/performance/results.js217 218 - name: Reset to original commit219 run: git reset --hard $GITHUB_SHA220 221 - name: Set up Node.js222 uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2223 with:224 node-version-file: '.nvmrc'225 cache: npm226 227 - name: Install npm dependencies228 run: npm ci229 230 258 - name: Set the environment to the baseline version 259 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} 231 260 run: | 232 261 npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }} … … 234 263 235 264 - name: Run any database upgrades 265 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} 236 266 run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }} 237 267 268 - name: Flush cache 269 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} 270 run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }} 271 272 - name: Delete expired transients 273 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} 274 run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }} 275 238 276 - name: Run baseline performance tests 277 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} 239 278 env: 240 279 TEST_RESULTS_PREFIX: base 241 280 run: npm run test:performance 242 281 243 - name: Print baseline performance tests results 244 env: 245 TEST_RESULTS_PREFIX: base 246 run: node ./tests/performance/results.js 247 248 - name: Compare results with base 282 - name: Archive artifacts 283 uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 284 if: always() 285 with: 286 name: performance-artifacts${{ matrix.memcached && '-memcached' || '' }}-${{ github.run_id }} 287 path: artifacts 288 if-no-files-found: ignore 289 290 - name: Compare results 249 291 run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md 250 292 … … 254 296 - name: Set the base sha 255 297 # Only needed when publishing results. 256 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}298 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }} 257 299 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 258 300 id: base-sha … … 265 307 - name: Set commit details 266 308 # Only needed when publishing results. 267 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}309 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }} 268 310 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 269 311 id: commit-timestamp … … 276 318 - name: Publish performance results 277 319 # Only publish results on pushes to trunk. 278 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}320 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }} 279 321 env: 280 322 BASE_SHA: ${{ steps.base-sha.outputs.result }} -
trunk/tests/performance/compare-results.js
r57083 r58076 4 4 * External dependencies. 5 5 */ 6 const fs= require( 'node:fs' );7 const path= require( 'node:path' );6 const { readFileSync, writeFileSync, existsSync } = require( 'node:fs' ); 7 const { join } = require( 'node:path' ); 8 8 9 9 /** 10 10 * Internal dependencies 11 11 */ 12 const { median } = require( './utils' ); 12 const { 13 median, 14 formatAsMarkdownTable, 15 formatValue, 16 linkToSha, 17 standardDeviation, 18 medianAbsoluteDeviation, 19 accumulateValues, 20 } = require( './utils' ); 21 22 process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' ); 23 24 const args = process.argv.slice( 2 ); 25 const summaryFile = args[ 0 ]; 13 26 14 27 /** … … 16 29 * 17 30 * @param {string} fileName The name of the file. 18 * @return s An array of parsed objects from each file.31 * @return {Array<{file: string, title: string, results: Record<string,number[]>[]}>} Parsed object. 19 32 */ 20 const parseFile = ( fileName ) => 21 JSON.parse( 22 fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' ) 23 ); 24 25 // The list of test suites to log. 26 const testSuites = [ 27 'admin', 28 'admin-l10n', 29 'home-block-theme', 30 'home-block-theme-l10n', 31 'home-classic-theme', 32 'home-classic-theme-l10n', 33 ]; 34 35 // The current commit's results. 36 const testResults = Object.fromEntries( 37 testSuites 38 .filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `${ key }.test.results.json` ) ) ) 39 .map( ( key ) => [ key, parseFile( `${ key }.test.results.json` ) ] ) 40 ); 41 42 // The previous commit's results. 43 const prevResults = Object.fromEntries( 44 testSuites 45 .filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `before-${ key }.test.results.json` ) ) ) 46 .map( ( key ) => [ key, parseFile( `before-${ key }.test.results.json` ) ] ) 47 ); 48 49 const args = process.argv.slice( 2 ); 50 51 const summaryFile = args[ 0 ]; 52 53 /** 54 * Formats an array of objects as a Markdown table. 55 * 56 * For example, this array: 57 * 58 * [ 59 * { 60 * foo: 123, 61 * bar: 456, 62 * baz: 'Yes', 63 * }, 64 * { 65 * foo: 777, 66 * bar: 999, 67 * baz: 'No', 68 * } 69 * ] 70 * 71 * Will result in the following table: 72 * 73 * | foo | bar | baz | 74 * |-----|-----|-----| 75 * | 123 | 456 | Yes | 76 * | 777 | 999 | No | 77 * 78 * @param {Array<Object>} rows Table rows. 79 * @returns {string} Markdown table content. 80 */ 81 function formatAsMarkdownTable( rows ) { 82 let result = ''; 83 const headers = Object.keys( rows[ 0 ] ); 84 for ( const header of headers ) { 85 result += `| ${ header } `; 86 } 87 result += '|\n'; 88 for ( const header of headers ) { 89 result += '| ------ '; 90 } 91 result += '|\n'; 92 93 for ( const row of rows ) { 94 for ( const value of Object.values( row ) ) { 95 result += `| ${ value } `; 96 } 97 result += '|\n'; 33 function parseFile( fileName ) { 34 const file = join( process.env.WP_ARTIFACTS_PATH, fileName ); 35 if ( ! existsSync( file ) ) { 36 return []; 98 37 } 99 38 100 return result;39 return JSON.parse( readFileSync( file, 'utf8' ) ); 101 40 } 102 41 103 42 /** 104 * Returns a Markdown link to a Git commit on the current GitHub repository. 105 * 106 * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e` 107 * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a]. 108 * 109 * @param {string} sha Commit SHA. 110 * @return string Link 43 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>} 111 44 */ 112 function linkToSha(sha) { 113 const repoName = process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop'; 45 const beforeStats = parseFile( 'before-performance-results.json' ); 114 46 115 return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`; 47 /** 48 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>} 49 */ 50 const afterStats = parseFile( 'performance-results.json' ); 51 52 let summaryMarkdown = `## Performance Test Results\n\n`; 53 54 if ( process.env.TARGET_SHA ) { 55 if ( beforeStats.length > 0 ) { 56 if (process.env.GITHUB_SHA) { 57 summaryMarkdown += `This compares the results from this commit (${linkToSha( 58 process.env.GITHUB_SHA 59 )}) with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`; 60 } else { 61 summaryMarkdown += `This compares the results from this commit with the ones from ${linkToSha( 62 process.env.TARGET_SHA 63 )}.\n\n`; 64 } 65 } else { 66 summaryMarkdown += `Note: no build was found for the target commit ${linkToSha(process.env.TARGET_SHA)}. No comparison is possible.\n\n`; 67 } 116 68 } 117 69 118 let summaryMarkdown = `# Performance Test Results\n\n`; 70 const numberOfRepetitions = afterStats[ 0 ].results.length; 71 const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ] 72 .length; 119 73 120 if ( process.env.GITHUB_SHA ) { 121 summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`; 122 } else { 123 summaryMarkdown += `🛎️ Performance test results are in!\n\n`; 124 } 74 const repetitions = `${ numberOfRepetitions } ${ 75 numberOfRepetitions === 1 ? 'repetition' : 'repetitions' 76 }`; 77 const iterations = `${ numberOfIterations } ${ 78 numberOfIterations === 1 ? 'iteration' : 'iterations' 79 }`; 125 80 126 if ( process.env.TARGET_SHA ) { 127 summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`; 128 } 81 summaryMarkdown += `All numbers are median values over ${ repetitions } with ${ iterations } each.\n\n`; 129 82 130 83 if ( process.env.GITHUB_SHA ) { … … 134 87 console.log( 'Performance Test Results\n' ); 135 88 136 console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' ); 89 console.log( 90 `All numbers are median values over ${ repetitions } with ${ iterations } each.\n` 91 ); 137 92 138 /** 139 * Nicely formats a given value. 140 * 141 * @param {string} metric Metric. 142 * @param {number} value 143 */ 144 function formatValue( metric, value) { 145 if ( null === value ) { 146 return 'N/A'; 147 } 148 if ( 'wpMemoryUsage' === metric ) { 149 return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`; 150 } 151 152 return `${ value.toFixed( 2 ) } ms`; 93 if ( process.env.GITHUB_SHA ) { 94 console.log( 95 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' 96 ); 153 97 } 154 98 155 for ( const key of testSuites ) { 156 const current = testResults[ key ] || {}; 157 const prev = prevResults[ key ] || {}; 99 for ( const { title, results } of afterStats ) { 100 const prevStat = beforeStats.find( ( s ) => s.title === title ); 158 101 159 const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace( 160 /-+/g, 161 ' ' 162 ); 163 102 /** 103 * @type {Array<Record<string, string>>} 104 */ 164 105 const rows = []; 165 106 166 for ( const [ metric, values ] of Object.entries( current ) ) { 107 const newResults = accumulateValues( results ); 108 // Only do comparison if the number of results is the same. 109 const prevResults = 110 prevStat && prevStat.results.length === results.length 111 ? accumulateValues( prevStat.results ) 112 : {}; 113 114 for ( const [ metric, values ] of Object.entries( newResults ) ) { 115 const prevValues = prevResults[ metric ] ? prevResults[ metric ] : null; 116 167 117 const value = median( values ); 168 const prevValue = prev[ metric ] ? median( prev[ metric ] ) : null; 118 const prevValue = prevValues ? median( prevValues ) : 0; 119 const delta = value - prevValue; 120 const percentage = ( delta / value ) * 100; 121 const showDiff = 122 metric !== 'wpExtObjCache' && ! Number.isNaN( percentage ); 169 123 170 const delta = null !== prevValue ? value - prevValue : 0171 const percentage = ( delta / value ) * 100;172 124 rows.push( { 173 125 Metric: metric, 174 Before: formatValue( metric, prevValue ),126 Before: prevValues ? formatValue( metric, prevValue ) : 'N/A', 175 127 After: formatValue( metric, value ), 176 'Diff abs.': formatValue( metric, delta ), 177 'Diff %': `${ percentage.toFixed( 2 ) } %`, 128 'Diff abs.': showDiff ? formatValue( metric, delta ) : '', 129 'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '', 130 STD: showDiff 131 ? formatValue( metric, standardDeviation( values ) ) 132 : '', 133 MAD: showDiff 134 ? formatValue( metric, medianAbsoluteDeviation( values ) ) 135 : '', 178 136 } ); 179 137 } 180 138 139 console.log( title ); 181 140 if ( rows.length > 0 ) { 182 summaryMarkdown += `## ${ title }\n\n`; 183 summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`; 141 console.table( rows ); 142 } else { 143 console.log( '(no results)' ); 144 } 184 145 185 console.log( title ); 186 console.table( rows ); 187 } 146 summaryMarkdown += `**${ title }**\n\n`; 147 summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`; 188 148 } 189 149 150 writeFileSync( 151 join( process.env.WP_ARTIFACTS_PATH, '/performance-results.md' ), 152 summaryMarkdown 153 ); 154 190 155 if ( summaryFile ) { 191 fs.writeFileSync( 192 summaryFile, 193 summaryMarkdown 194 ); 156 writeFileSync( summaryFile, summaryMarkdown ); 195 157 } -
trunk/tests/performance/config/global-setup.js
r56926 r58076 31 31 32 32 // Reset the test environment before running the tests. 33 await Promise.all( [ 34 requestUtils.activateTheme( 'twentytwentyone' ), 35 ] ); 33 await Promise.all( [ requestUtils.activateTheme( 'twentytwentyone' ) ] ); 36 34 37 35 await requestContext.dispose(); -
trunk/tests/performance/config/performance-reporter.js
r56926 r58076 2 2 * External dependencies 3 3 */ 4 import { join, dirname, basename } from 'node:path'; 5 import { writeFileSync } from 'node:fs'; 6 7 /** 8 * Internal dependencies 9 */ 10 import { getResultsFilename } from '../utils'; 4 import { join } from 'node:path'; 5 import { writeFileSync, existsSync, mkdirSync } from 'node:fs'; 11 6 12 7 /** … … 15 10 class PerformanceReporter { 16 11 /** 12 * 13 * @type {Record<string,{title: string; results: Record< string, number[] >[];}>} 14 */ 15 allResults = {}; 16 17 /** 18 * Called after a test has been finished in the worker process. 19 * 20 * Used to add test results to the final summary of all tests. 17 21 * 18 22 * @param {import('@playwright/test/reporter').TestCase} test … … 25 29 26 30 if ( performanceResults?.body ) { 27 writeFileSync( 28 join( 29 dirname( test.location.file ), 30 getResultsFilename( basename( test.location.file, '.js' ) ) 31 ), 32 performanceResults.body.toString( 'utf-8' ) 31 // 0 = empty, 1 = browser, 2 = file name, 3 = test suite name, 4 = test name. 32 const titlePath = test.titlePath(); 33 const title = `${ titlePath[ 3 ] } › ${ titlePath[ 4 ] }`; 34 35 // results is an array in case repeatEach is > 1. 36 37 this.allResults[ title ] ??= { 38 file: test.location.file, // Unused, but useful for debugging. 39 results: [], 40 }; 41 42 this.allResults[ title ].results.push( 43 JSON.parse( performanceResults.body.toString( 'utf-8' ) ) 33 44 ); 34 45 } 46 } 47 48 /** 49 * Called after all tests have been run, or testing has been interrupted. 50 * 51 * Writes all raw numbers to a file for further processing, 52 * for example to compare with a previous run. 53 * 54 * @param {import('@playwright/test/reporter').FullResult} result 55 */ 56 onEnd( result ) { 57 const summary = []; 58 59 for ( const [ title, { file, results } ] of Object.entries( 60 this.allResults 61 ) ) { 62 summary.push( { 63 file, 64 title, 65 results, 66 } ); 67 } 68 69 if ( ! existsSync( process.env.WP_ARTIFACTS_PATH ) ) { 70 mkdirSync( process.env.WP_ARTIFACTS_PATH ); 71 } 72 73 const prefix = process.env.TEST_RESULTS_PREFIX; 74 const fileNamePrefix = prefix ? `${ prefix }-` : ''; 75 76 writeFileSync( 77 join( 78 process.env.WP_ARTIFACTS_PATH, 79 `${ fileNamePrefix }performance-results.json` 80 ), 81 JSON.stringify( summary, null, 2 ) 82 ); 35 83 } 36 84 } -
trunk/tests/performance/log-results.js
r57083 r58076 1 1 #!/usr/bin/env node 2 3 /* 4 * Get the test results and format them in the way required by the API. 5 * 6 * Contains some backward compatibility logic for the original test suite format, 7 * see #59900 for details. 8 */ 2 9 3 10 /** 4 11 * External dependencies. 5 12 */ 6 const fs = require( 'fs' );7 const path = require( 'path' );8 13 const https = require( 'https' ); 9 const [ token, branch, hash, baseHash, timestamp, host ] = process.argv.slice( 2 ); 10 const { median } = require( './utils' ); 14 const [ token, branch, hash, baseHash, timestamp, host ] = 15 process.argv.slice( 2 ); 16 const { median, parseFile, accumulateValues } = require( './utils' ); 11 17 12 // The list of test suites to log. 13 const testSuites = [ 14 'admin', 15 'admin-l10n', 16 'home-block-theme', 17 'home-block-theme-l10n', 18 'home-classic-theme', 19 'home-classic-theme-l10n', 20 ]; 21 22 // A list of results to parse based on test suites. 23 const testResults = testSuites.map(( key ) => ({ 24 key, 25 file: `${ key }.test.results.json`, 26 })); 27 28 // A list of base results to parse based on test suites. 29 const baseResults = testSuites.map(( key ) => ({ 30 key, 31 file: `base-${ key }.test.results.json`, 32 })); 18 const testSuiteMap = { 19 'Admin › Locale: en_US': 'admin', 20 'Admin › Locale: de_DE': 'admin-l10n', 21 'Front End › Theme: twentytwentyone, Locale: en_US': 'home-classic-theme', 22 'Front End › Theme: twentytwentyone, Locale: de_DE': 23 'home-classic-theme-l10n', 24 'Front End › Theme: twentytwentythree, Locale: en_US': 'home-block-theme', 25 'Front End › Theme: twentytwentythree, Locale: de_DE': 26 'home-block-theme-l10n', 27 }; 33 28 34 29 /** 35 * Parse test files into JSON objects. 36 * 37 * @param {string} fileName The name of the file. 38 * @returns An array of parsed objects from each file. 30 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>} 39 31 */ 40 const parseFile = ( fileName ) => ( 41 JSON.parse( 42 fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' ) 43 ) 44 ); 32 const afterStats = parseFile( 'performance-results.json' ); 33 34 /** 35 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>} 36 */ 37 const baseStats = parseFile( 'base-performance-results.json' ); 38 39 /** 40 * @type {Record<string, number>} 41 */ 42 const metrics = {}; 43 /** 44 * @type {Record<string, number>} 45 */ 46 const baseMetrics = {}; 47 48 for ( const { title, results } of afterStats ) { 49 const testSuiteName = testSuiteMap[ title ]; 50 if ( ! testSuiteName ) { 51 continue; 52 } 53 54 const baseStat = baseStats.find( ( s ) => s.title === title ); 55 56 const currResults = accumulateValues( results ); 57 const baseResults = accumulateValues( baseStat.results ); 58 59 for ( const [ metric, values ] of Object.entries( currResults ) ) { 60 metrics[ `${ testSuiteName }-${ metric }` ] = median( values ); 61 } 62 63 for ( const [ metric, values ] of Object.entries( baseResults ) ) { 64 baseMetrics[ `${ testSuiteName }-${ metric }` ] = median( values ); 65 } 66 } 67 68 process.exit( 0 ); 45 69 46 70 /** … … 48 72 * 49 73 * @param {Object[]} results A list of results to format. 50 * @return {Object []} Metrics.74 * @return {Object} Metrics. 51 75 */ 52 76 const formatResults = ( results ) => { 53 return results.reduce( 54 ( result, { key, file } ) => { 55 return { 56 ...result, 57 ...Object.fromEntries( 58 Object.entries( 59 parseFile( file ) ?? {} 60 ).map( ( [ metric, value ] ) => [ 77 return results.reduce( ( result, { key, file } ) => { 78 return { 79 ...result, 80 ...Object.fromEntries( 81 Object.entries( parseFile( file ) ?? {} ).map( 82 ( [ metric, value ] ) => [ 61 83 key + '-' + metric, 62 median ( value ), 63 ] ) 64 ), 65 }; 66 }, 67 {} 68 ); 84 median( value ), 85 ] 86 ) 87 ), 88 }; 89 }, {} ); 69 90 }; 70 91 -
trunk/tests/performance/playwright.config.js
r57083 r58076 24 24 workers: 1, 25 25 retries: 0, 26 repeatEach: 2, 26 27 timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 600_000, // Defaults to 10 minutes. 27 28 // Don't report slow test "files", as we will be running our tests in serial. 28 29 reportSlowTests: null, 30 preserveOutput: 'never', 29 31 webServer: { 30 32 ...baseConfig.webServer, … … 38 40 39 41 export default config; 40 -
trunk/tests/performance/specs/admin.test.js
r57083 r58076 13 13 }; 14 14 15 const locales = [ 'en_US', 'de_DE' ]; 16 15 17 test.describe( 'Admin', () => { 16 test.beforeAll( async ( { requestUtils } ) => { 17 await requestUtils.activateTheme( 'twentytwentyone' ); 18 } ); 18 for ( const locale of locales ) { 19 test.describe( `Locale: ${ locale }`, () => { 20 test.beforeAll( async ( { requestUtils } ) => { 21 await requestUtils.activateTheme( 'twentytwentyone' ); 22 await requestUtils.updateSiteSettings( { 23 language: 'en_US' === locale ? '' : locale, 24 } ); 25 } ); 19 26 20 test.afterAll( async ( {}, testInfo ) => { 21 await testInfo.attach( 'results', { 22 body: JSON.stringify( results, null, 2 ), 23 contentType: 'application/json', 24 } ); 25 } ); 27 test.afterAll( async ( { requestUtils }, testInfo ) => { 28 await testInfo.attach( 'results', { 29 body: JSON.stringify( results, null, 2 ), 30 contentType: 'application/json', 31 } ); 26 32 27 const iterations = Number( process.env.TEST_RUNS ); 28 for ( let i = 1; i <= iterations; i++ ) { 29 test( `Measure load time metrics (${ i } of ${ iterations })`, async ( { 30 admin, 31 metrics, 32 } ) => { 33 await admin.visitAdminPage( '/' ); 33 await requestUtils.updateSiteSettings( { 34 language: '', 35 } ); 34 36 35 const serverTiming = await metrics.getServerTiming(); 37 results.timeToFirstByte = []; 38 } ); 36 39 37 for ( const [ key, value ] of Object.entries( serverTiming ) ) { 38 results[ camelCaseDashes( key ) ] ??= []; 39 results[ camelCaseDashes( key ) ].push( value ); 40 test.afterAll( async ( {}, testInfo ) => { 41 await testInfo.attach( 'results', { 42 body: JSON.stringify( results, null, 2 ), 43 contentType: 'application/json', 44 } ); 45 } ); 46 47 const iterations = Number( process.env.TEST_RUNS ); 48 for ( let i = 1; i <= iterations; i++ ) { 49 test( `Measure load time metrics (${ i } of ${ iterations })`, async ( { 50 admin, 51 metrics, 52 } ) => { 53 await admin.visitAdminPage( '/' ); 54 55 const serverTiming = await metrics.getServerTiming(); 56 57 for ( const [ key, value ] of Object.entries( 58 serverTiming 59 ) ) { 60 results[ camelCaseDashes( key ) ] ??= []; 61 results[ camelCaseDashes( key ) ].push( value ); 62 } 63 64 const ttfb = await metrics.getTimeToFirstByte(); 65 results.timeToFirstByte.push( ttfb ); 66 } ); 40 67 } 41 42 const ttfb = await metrics.getTimeToFirstByte();43 results.timeToFirstByte.push( ttfb );44 68 } ); 45 69 } -
trunk/tests/performance/utils.js
r56928 r58076 1 /** 2 * External dependencies. 3 */ 4 const { readFileSync, existsSync } = require( 'node:fs' ); 5 const { join } = require( 'node:path' ); 6 7 process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' ); 8 9 /** 10 * Parse test files into JSON objects. 11 * 12 * @param {string} fileName The name of the file. 13 * @return {Array<{file: string, title: string, results: Record<string,number[]>[]}>} Parsed object. 14 */ 15 function parseFile( fileName ) { 16 const file = join( process.env.WP_ARTIFACTS_PATH, fileName ); 17 if ( ! existsSync( file ) ) { 18 return []; 19 } 20 21 return JSON.parse( readFileSync( file, 'utf8' ) ); 22 } 23 1 24 /** 2 25 * Computes the median number from an array numbers. … … 14 37 } 15 38 16 /**17 * Gets the result file name.18 *19 * @param {string} fileName File name.20 *21 * @return {string} Result file name.22 */23 function getResultsFilename( fileName ) {24 const prefix = process.env.TEST_RESULTS_PREFIX;25 const fileNamePrefix = prefix ? `${ prefix }-` : '';26 return `${fileNamePrefix + fileName}.results.json`;27 }28 29 39 function camelCaseDashes( str ) { 30 return str.replace( /-([a-z])/g, function ( g ) {40 return str.replace( /-([a-z])/g, function ( g ) { 31 41 return g[ 1 ].toUpperCase(); 32 42 } ); 33 43 } 34 44 45 /** 46 * Formats an array of objects as a Markdown table. 47 * 48 * For example, this array: 49 * 50 * [ 51 * { 52 * foo: 123, 53 * bar: 456, 54 * baz: 'Yes', 55 * }, 56 * { 57 * foo: 777, 58 * bar: 999, 59 * baz: 'No', 60 * } 61 * ] 62 * 63 * Will result in the following table: 64 * 65 * | foo | bar | baz | 66 * |-----|-----|-----| 67 * | 123 | 456 | Yes | 68 * | 777 | 999 | No | 69 * 70 * @param {Array<Object>} rows Table rows. 71 * @returns {string} Markdown table content. 72 */ 73 function formatAsMarkdownTable( rows ) { 74 let result = ''; 75 76 if ( ! rows.length ) { 77 return result; 78 } 79 80 const headers = Object.keys( rows[ 0 ] ); 81 for ( const header of headers ) { 82 result += `| ${ header } `; 83 } 84 result += '|\n'; 85 for ( const header of headers ) { 86 result += '| ------ '; 87 } 88 result += '|\n'; 89 90 for ( const row of rows ) { 91 for ( const value of Object.values( row ) ) { 92 result += `| ${ value } `; 93 } 94 result += '|\n'; 95 } 96 97 return result; 98 } 99 100 /** 101 * Nicely formats a given value. 102 * 103 * @param {string} metric Metric. 104 * @param {number} value 105 */ 106 function formatValue( metric, value ) { 107 if ( null === value ) { 108 return 'N/A'; 109 } 110 111 if ( 'wpMemoryUsage' === metric ) { 112 return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`; 113 } 114 115 if ( 'wpExtObjCache' === metric ) { 116 return 1 === value ? 'yes' : 'no'; 117 } 118 119 if ( 'wpDbQueries' === metric ) { 120 return value; 121 } 122 123 return `${ value.toFixed( 2 ) } ms`; 124 } 125 126 /** 127 * Returns a Markdown link to a Git commit on the current GitHub repository. 128 * 129 * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e` 130 * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a]. 131 * 132 * @param {string} sha Commit SHA. 133 * @return string Link 134 */ 135 function linkToSha( sha ) { 136 const repoName = 137 process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop'; 138 139 return `[${ sha.slice( 140 0, 141 7 142 ) }](https://github.com/${ repoName }/commit/${ sha })`; 143 } 144 145 function standardDeviation( array = [] ) { 146 if ( ! array.length ) { 147 return 0; 148 } 149 150 const mean = array.reduce( ( a, b ) => a + b ) / array.length; 151 return Math.sqrt( 152 array 153 .map( ( x ) => Math.pow( x - mean, 2 ) ) 154 .reduce( ( a, b ) => a + b ) / array.length 155 ); 156 } 157 158 function medianAbsoluteDeviation( array = [] ) { 159 if ( ! array.length ) { 160 return 0; 161 } 162 163 const med = median( array ); 164 return median( array.map( ( a ) => Math.abs( a - med ) ) ); 165 } 166 167 /** 168 * 169 * @param {Array<Record<string, number[]>>} results 170 * @returns {Record<string, number[]>} 171 */ 172 function accumulateValues( results ) { 173 return results.reduce( ( acc, result ) => { 174 for ( const [ metric, values ] of Object.entries( result ) ) { 175 acc[ metric ] = acc[ metric ] ?? []; 176 acc[ metric ].push( ...values ); 177 } 178 return acc; 179 }, {} ); 180 } 181 35 182 module.exports = { 183 parseFile, 36 184 median, 37 getResultsFilename,38 185 camelCaseDashes, 186 formatAsMarkdownTable, 187 formatValue, 188 linkToSha, 189 standardDeviation, 190 medianAbsoluteDeviation, 191 accumulateValues, 39 192 }; -
trunk/tests/performance/wp-content/mu-plugins/server-timing.php
r57083 r58076 5 5 static function ( $template ) { 6 6 7 global $timestart ;7 global $timestart, $wpdb; 8 8 9 9 $server_timing_values = array(); … … 16 16 add_action( 17 17 'shutdown', 18 static function () use ( $server_timing_values, $template_start ) { 19 20 global $timestart; 21 18 static function () use ( $server_timing_values, $template_start, $wpdb ) { 22 19 $output = ob_get_clean(); 23 20 … … 31 28 * This is a nice little trick as it allows to easily get this information in JS. 32 29 */ 33 $server_timing_values['memory-usage'] = memory_get_usage(); 30 $server_timing_values['memory-usage'] = memory_get_usage(); 31 $server_timing_values['db-queries'] = $wpdb->num_queries; 32 $server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0; 34 33 35 34 $header_values = array(); … … 51 50 PHP_INT_MAX 52 51 ); 52 53 add_action( 54 'admin_init', 55 static function () { 56 global $timestart, $wpdb; 57 58 ob_start(); 59 60 add_action( 61 'shutdown', 62 static function () use ( $wpdb, $timestart ) { 63 $output = ob_get_clean(); 64 65 $server_timing_values = array(); 66 67 $server_timing_values['total'] = microtime( true ) - $timestart; 68 69 /* 70 * While values passed via Server-Timing are intended to be durations, 71 * any numeric value can actually be passed. 72 * This is a nice little trick as it allows to easily get this information in JS. 73 */ 74 $server_timing_values['memory-usage'] = memory_get_usage(); 75 $server_timing_values['db-queries'] = $wpdb->num_queries; 76 $server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0; 77 78 $header_values = array(); 79 foreach ( $server_timing_values as $slug => $value ) { 80 if ( is_float( $value ) ) { 81 $value = round( $value * 1000.0, 2 ); 82 } 83 $header_values[] = sprintf( 'wp-%1$s;dur=%2$s', $slug, $value ); 84 } 85 header( 'Server-Timing: ' . implode( ', ', $header_values ) ); 86 87 echo $output; 88 }, 89 PHP_INT_MIN 90 ); 91 }, 92 PHP_INT_MAX 93 );
Note: See TracChangeset
for help on using the changeset viewer.