Make WordPress Core

Changeset 58076


Ignore:
Timestamp:
05/02/2024 01:57:49 PM (9 months ago)
Author:
swissspidy
Message:

Build/Test Tools: Overhaul performance tests to improve stability and cover more scenarios.

Simplifies the tests setup by leveraging a test matrix, improving maintenance and making it much easier to test more scenarios. With this change, tests are now also run with an external object cache (Memcached). Additional information such as memory usage and the number of database queries is now collected as well.

Improves test setup and cleanup by disabling external HTTP requests and cron for the tests, as well as deleting expired transients and flushing the cache in-between. This should aid the test stability.

When testing the previous commit / target branch, this now leverages the already built artifact from the build process workflow. Raw test results are now also uploaded as artifacts to aid debugging.

Props swissspidy, adamsilverstein, joemcgill, mukesh27, desrosj, youknowriad, flixos90.
Fixes #59900

Location:
trunk
Files:
1 added
6 deleted
9 edited

Legend:

Unmodified
Added
Removed
  • trunk/.github/workflows/performance.yml

    r57918 r58076  
    6767  # - Install WordPress Importer plugin.
    6868  # - Import mock data.
     69  # - Deactivate WordPress Importer plugin.
    6970  # - Update permalink structure.
     71  # - Install additional languages.
     72  # - Disable external HTTP requests.
     73  # - Disable cron.
     74  # - List defined constants.
    7075  # - Install MU plugin.
    7176  # - Run performance tests (current commit).
    72   # - Print performance tests results.
    73   # - Check out target commit (target branch or previous commit).
    74   # - Switch Node.js versions if necessary.
    75   # - Install npm dependencies.
    76   # - Build WordPress.
     77  # - Download previous build artifact (target branch or previous commit).
     78  # - Download artifact.
     79  # - Unzip the build.
    7780  # - Run any database upgrades.
     81  # - Flush cache.
     82  # - Delete expired transients.
    7883  # - Run performance tests (previous/target commit).
    79   # - Print target performance tests results.
    80   # - Reset to original commit.
    81   # - Switch Node.js versions if necessary.
    82   # - Install npm dependencies.
    8384  # - Set the environment to the baseline version.
    8485  # - Run any database upgrades.
     86  # - Flush cache.
     87  # - Delete expired transients.
    8588  # - Run baseline performance tests.
    86   # - Print baseline performance tests results.
    87   # - Compare results with base.
     89  # - Archive artifacts.
     90  # - Compare results.
    8891  # - Add workflow summary.
    8992  # - Set the base sha.
     
    9194  # - Publish performance results.
    9295  # - Ensure version-controlled files are not modified or deleted.
    93   # - Dispatch workflow run.
    9496  performance:
    95     name: Run performance tests
     97    name: Run performance tests ${{ matrix.memcached && '(with memcached)' || '' }}
    9698    runs-on: ubuntu-latest
    9799    permissions:
    98100      contents: read
    99101    if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) && ! contains( github.event.before, '00000000' ) }}
    100 
     102    strategy:
     103      fail-fast: false
     104      matrix:
     105        memcached: [ true, false ]
     106    env:
     107      LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
    101108    steps:
    102109      - name: Configure environment variables
     
    128135
    129136      - name: Install Playwright browsers
    130         run: npx playwright install --with-deps
     137        run: npx playwright install --with-deps chromium
    131138
    132139      - name: Build WordPress
     
    134141
    135142      - name: Start Docker environment
    136         run: |
    137           npm run env:start
     143        run: npm run env:start
     144
     145      - name: Install object cache drop-in
     146        if: ${{ matrix.memcached }}
     147        run: cp src/wp-content/object-cache.php build/wp-content/object-cache.php
    138148
    139149      - name: Log running Docker containers
     
    161171          rm themeunittestdata.wordpress.xml
    162172
     173      - name: Deactivate WordPress Importer plugin
     174        run: npm run env:cli -- plugin deactivate wordpress-importer --path=/var/www/${{ env.LOCAL_DIR }}
     175
    163176      - name: Update permalink structure
    164         run: |
    165           npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
     177        run: npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
    166178
    167179      - name: Install additional languages
     
    171183          npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
    172184
     185      # Prevent background update checks from impacting test stability.
     186      - name: Disable external HTTP requests
     187        run: npm run env:cli -- config set WP_HTTP_BLOCK_EXTERNAL true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
     188
     189      # Prevent background tasks from impacting test stability.
     190      - name: Disable cron
     191        run: npm run env:cli -- config set DISABLE_WP_CRON true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
     192
     193      - name: List defined constants
     194        run: npm run env:cli -- config list --path=/var/www/${{ env.LOCAL_DIR }}
     195
    173196      - name: Install MU plugin
    174197        run: |
     
    179202        run: npm run test:performance
    180203
    181       - name: Print performance tests results
    182         run: node ./tests/performance/results.js
    183 
    184       - name: Check out target commit (target branch or previous commit)
    185         run: |
    186           if [[ -z "$TARGET_REF" ]]; then
    187             git fetch -n origin $TARGET_SHA
    188           else
    189             git fetch -n origin $TARGET_REF
    190           fi
    191           git reset --hard $TARGET_SHA
    192 
    193       - name: Set up Node.js
    194         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
    195         with:
    196           node-version-file: '.nvmrc'
    197           cache: npm
    198 
    199       - name: Install npm dependencies
    200         run: npm ci
    201 
    202       - name: Build WordPress
    203         run: npm run build
     204      - name: Download previous build artifact (target branch or previous commit)
     205        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
     206        id: get-previous-build
     207        with:
     208          script: |
     209            const artifacts = await github.rest.actions.listArtifactsForRepo({
     210              owner: context.repo.owner,
     211              repo: context.repo.repo,
     212              name: 'wordpress-build-' + process.env.TARGET_SHA,
     213            });
     214
     215            const matchArtifact = artifacts.data.artifacts[0];
     216
     217            if ( ! matchArtifact ) {
     218              core.setFailed( 'No artifact found!' );
     219              return false;
     220            }
     221
     222            const download = await github.rest.actions.downloadArtifact( {
     223              owner: context.repo.owner,
     224              repo: context.repo.repo,
     225              artifact_id: matchArtifact.id,
     226              archive_format: 'zip',
     227            } );
     228
     229            const fs = require( 'fs' );
     230            fs.writeFileSync( '${{ github.workspace }}/before.zip', Buffer.from( download.data ) )
     231
     232            return true;
     233
     234      - name: Unzip the build
     235        if: ${{ steps.get-previous-build.outputs.result }}
     236        run: |
     237          unzip ${{ github.workspace }}/before.zip
     238          unzip -o ${{ github.workspace }}/wordpress.zip
    204239
    205240      - name: Run any database upgrades
     241        if: ${{ steps.get-previous-build.outputs.result }}
    206242        run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
    207243
    208       - name: Run target performance tests (base/previous commit)
     244      - name: Flush cache
     245        if: ${{ steps.get-previous-build.outputs.result }}
     246        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
     247
     248      - name: Delete expired transients
     249        if: ${{ steps.get-previous-build.outputs.result }}
     250        run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
     251
     252      - name: Run target performance tests (previous/target commit)
     253        if: ${{ steps.get-previous-build.outputs.result }}
    209254        env:
    210255          TEST_RESULTS_PREFIX: before
    211256        run: npm run test:performance
    212257
    213       - name: Print target performance tests results
    214         env:
    215           TEST_RESULTS_PREFIX: before
    216         run: node ./tests/performance/results.js
    217 
    218       - name: Reset to original commit
    219         run: git reset --hard $GITHUB_SHA
    220 
    221       - name: Set up Node.js
    222         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
    223         with:
    224           node-version-file: '.nvmrc'
    225           cache: npm
    226 
    227       - name: Install npm dependencies
    228         run: npm ci
    229 
    230258      - name: Set the environment to the baseline version
     259        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
    231260        run: |
    232261          npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
     
    234263
    235264      - name: Run any database upgrades
     265        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
    236266        run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
    237267
     268      - name: Flush cache
     269        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     270        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
     271
     272      - name: Delete expired transients
     273        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     274        run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
     275
    238276      - name: Run baseline performance tests
     277        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
    239278        env:
    240279          TEST_RESULTS_PREFIX: base
    241280        run: npm run test:performance
    242281
    243       - name: Print baseline performance tests results
    244         env:
    245           TEST_RESULTS_PREFIX: base
    246         run: node ./tests/performance/results.js
    247 
    248       - name: Compare results with base
     282      - name: Archive artifacts
     283        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
     284        if: always()
     285        with:
     286          name: performance-artifacts${{ matrix.memcached && '-memcached' || '' }}-${{ github.run_id }}
     287          path: artifacts
     288          if-no-files-found: ignore
     289
     290      - name: Compare results
    249291        run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
    250292
     
    254296      - name: Set the base sha
    255297        # Only needed when publishing results.
    256         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     298        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
    257299        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
    258300        id: base-sha
     
    265307      - name: Set commit details
    266308        # Only needed when publishing results.
    267         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     309        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
    268310        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
    269311        id: commit-timestamp
     
    276318      - name: Publish performance results
    277319        # Only publish results on pushes to trunk.
    278         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     320        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
    279321        env:
    280322            BASE_SHA: ${{ steps.base-sha.outputs.result }}
  • trunk/tests/performance/compare-results.js

    r57083 r58076  
    44 * External dependencies.
    55 */
    6 const fs = require( 'node:fs' );
    7 const path = require( 'node:path' );
     6const { readFileSync, writeFileSync, existsSync } = require( 'node:fs' );
     7const { join } = require( 'node:path' );
    88
    99/**
    1010 * Internal dependencies
    1111 */
    12 const { median } = require( './utils' );
     12const {
     13    median,
     14    formatAsMarkdownTable,
     15    formatValue,
     16    linkToSha,
     17    standardDeviation,
     18    medianAbsoluteDeviation,
     19    accumulateValues,
     20} = require( './utils' );
     21
     22process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
     23
     24const args = process.argv.slice( 2 );
     25const summaryFile = args[ 0 ];
    1326
    1427/**
     
    1629 *
    1730 * @param {string} fileName The name of the file.
    18  * @returns An array of parsed objects from each file.
     31 * @return {Array<{file: string, title: string, results: Record<string,number[]>[]}>} Parsed object.
    1932 */
    20 const parseFile = ( fileName ) =>
    21     JSON.parse(
    22         fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
    23     );
    24 
    25 // The list of test suites to log.
    26 const testSuites = [
    27     'admin',
    28     'admin-l10n',
    29     'home-block-theme',
    30     'home-block-theme-l10n',
    31     'home-classic-theme',
    32     'home-classic-theme-l10n',
    33 ];
    34 
    35 // The current commit's results.
    36 const testResults = Object.fromEntries(
    37     testSuites
    38         .filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `${ key }.test.results.json` ) ) )
    39         .map( ( key ) => [ key, parseFile( `${ key }.test.results.json` ) ] )
    40 );
    41 
    42 // The previous commit's results.
    43 const prevResults = Object.fromEntries(
    44     testSuites
    45         .filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `before-${ key }.test.results.json` ) ) )
    46         .map( ( key ) => [ key, parseFile( `before-${ key }.test.results.json` ) ] )
    47 );
    48 
    49 const args = process.argv.slice( 2 );
    50 
    51 const summaryFile = args[ 0 ];
    52 
    53 /**
    54  * Formats an array of objects as a Markdown table.
    55  *
    56  * For example, this array:
    57  *
    58  * [
    59  *  {
    60  *      foo: 123,
    61  *      bar: 456,
    62  *      baz: 'Yes',
    63  *  },
    64  *  {
    65  *      foo: 777,
    66  *      bar: 999,
    67  *      baz: 'No',
    68  *  }
    69  * ]
    70  *
    71  * Will result in the following table:
    72  *
    73  * | foo | bar | baz |
    74  * |-----|-----|-----|
    75  * | 123 | 456 | Yes |
    76  * | 777 | 999 | No  |
    77  *
    78  * @param {Array<Object>} rows Table rows.
    79  * @returns {string} Markdown table content.
    80  */
    81 function formatAsMarkdownTable( rows ) {
    82     let result = '';
    83     const headers = Object.keys( rows[ 0 ] );
    84     for ( const header of headers ) {
    85         result += `| ${ header } `;
    86     }
    87     result += '|\n';
    88     for ( const header of headers ) {
    89         result += '| ------ ';
    90     }
    91     result += '|\n';
    92 
    93     for ( const row of rows ) {
    94         for ( const value of Object.values( row ) ) {
    95             result += `| ${ value } `;
    96         }
    97         result += '|\n';
     33function parseFile( fileName ) {
     34    const file = join( process.env.WP_ARTIFACTS_PATH, fileName );
     35    if ( ! existsSync( file ) ) {
     36        return [];
    9837    }
    9938
    100     return result;
     39    return JSON.parse( readFileSync( file, 'utf8' ) );
    10140}
    10241
    10342/**
    104  * Returns a Markdown link to a Git commit on the current GitHub repository.
    105  *
    106  * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e`
    107  * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a].
    108  *
    109  * @param {string} sha Commit SHA.
    110  * @return string Link
     43 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
    11144 */
    112 function linkToSha(sha) {
    113     const repoName = process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop';
     45const beforeStats = parseFile( 'before-performance-results.json' );
    11446
    115     return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`;
     47/**
     48 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
     49 */
     50const afterStats = parseFile( 'performance-results.json' );
     51
     52let summaryMarkdown = `## Performance Test Results\n\n`;
     53
     54if ( process.env.TARGET_SHA ) {
     55    if ( beforeStats.length > 0 ) {
     56        if (process.env.GITHUB_SHA) {
     57            summaryMarkdown += `This compares the results from this commit (${linkToSha(
     58                process.env.GITHUB_SHA
     59            )}) with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
     60        } else {
     61            summaryMarkdown += `This compares the results from this commit with the ones from ${linkToSha(
     62                process.env.TARGET_SHA
     63            )}.\n\n`;
     64        }
     65    } else {
     66        summaryMarkdown += `Note: no build was found for the target commit ${linkToSha(process.env.TARGET_SHA)}. No comparison is possible.\n\n`;
     67    }
    11668}
    11769
    118 let summaryMarkdown = `# Performance Test Results\n\n`;
     70const numberOfRepetitions = afterStats[ 0 ].results.length;
     71const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ]
     72    .length;
    11973
    120 if ( process.env.GITHUB_SHA ) {
    121     summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`;
    122 } else {
    123     summaryMarkdown += `🛎️ Performance test results are in!\n\n`;
    124 }
     74const repetitions = `${ numberOfRepetitions } ${
     75    numberOfRepetitions === 1 ? 'repetition' : 'repetitions'
     76}`;
     77const iterations = `${ numberOfIterations } ${
     78    numberOfIterations === 1 ? 'iteration' : 'iterations'
     79}`;
    12580
    126 if ( process.env.TARGET_SHA ) {
    127     summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
    128 }
     81summaryMarkdown += `All numbers are median values over ${ repetitions } with ${ iterations } each.\n\n`;
    12982
    13083if ( process.env.GITHUB_SHA ) {
     
    13487console.log( 'Performance Test Results\n' );
    13588
    136 console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' );
     89console.log(
     90    `All numbers are median values over ${ repetitions } with ${ iterations } each.\n`
     91);
    13792
    138 /**
    139  * Nicely formats a given value.
    140  *
    141  * @param {string} metric Metric.
    142  * @param {number} value
    143  */
    144 function formatValue( metric, value) {
    145     if ( null === value ) {
    146         return 'N/A';
    147     }
    148     if ( 'wpMemoryUsage' === metric ) {
    149         return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
    150     }
    151 
    152     return `${ value.toFixed( 2 ) } ms`;
     93if ( process.env.GITHUB_SHA ) {
     94    console.log(
     95        'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n'
     96    );
    15397}
    15498
    155 for ( const key of testSuites ) {
    156     const current = testResults[ key ] || {};
    157     const prev = prevResults[ key ] || {};
     99for ( const { title, results } of afterStats ) {
     100    const prevStat = beforeStats.find( ( s ) => s.title === title );
    158101
    159     const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace(
    160         /-+/g,
    161         ' '
    162     );
    163 
     102    /**
     103     * @type {Array<Record<string, string>>}
     104     */
    164105    const rows = [];
    165106
    166     for ( const [ metric, values ] of Object.entries( current ) ) {
     107    const newResults = accumulateValues( results );
     108    // Only do comparison if the number of results is the same.
     109    const prevResults =
     110        prevStat && prevStat.results.length === results.length
     111            ? accumulateValues( prevStat.results )
     112            : {};
     113
     114    for ( const [ metric, values ] of Object.entries( newResults ) ) {
     115        const prevValues = prevResults[ metric ] ? prevResults[ metric ] : null;
     116
    167117        const value = median( values );
    168         const prevValue = prev[ metric ] ? median( prev[ metric ] ) : null;
     118        const prevValue = prevValues ? median( prevValues ) : 0;
     119        const delta = value - prevValue;
     120        const percentage = ( delta / value ) * 100;
     121        const showDiff =
     122            metric !== 'wpExtObjCache' && ! Number.isNaN( percentage );
    169123
    170         const delta = null !== prevValue ? value - prevValue : 0
    171         const percentage = ( delta / value ) * 100;
    172124        rows.push( {
    173125            Metric: metric,
    174             Before: formatValue( metric, prevValue ),
     126            Before: prevValues ? formatValue( metric, prevValue ) : 'N/A',
    175127            After: formatValue( metric, value ),
    176             'Diff abs.': formatValue( metric, delta ),
    177             'Diff %': `${ percentage.toFixed( 2 ) } %`,
     128            'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
     129            'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',
     130            STD: showDiff
     131                ? formatValue( metric, standardDeviation( values ) )
     132                : '',
     133            MAD: showDiff
     134                ? formatValue( metric, medianAbsoluteDeviation( values ) )
     135                : '',
    178136        } );
    179137    }
    180138
     139    console.log( title );
    181140    if ( rows.length > 0 ) {
    182         summaryMarkdown += `## ${ title }\n\n`;
    183         summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
     141        console.table( rows );
     142    } else {
     143        console.log( '(no results)' );
     144    }
    184145
    185         console.log( title );
    186         console.table( rows );
    187     }
     146    summaryMarkdown += `**${ title }**\n\n`;
     147    summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
    188148}
    189149
     150writeFileSync(
     151    join( process.env.WP_ARTIFACTS_PATH, '/performance-results.md' ),
     152    summaryMarkdown
     153);
     154
    190155if ( summaryFile ) {
    191     fs.writeFileSync(
    192         summaryFile,
    193         summaryMarkdown
    194     );
     156    writeFileSync( summaryFile, summaryMarkdown );
    195157}
  • trunk/tests/performance/config/global-setup.js

    r56926 r58076  
    3131
    3232    // Reset the test environment before running the tests.
    33     await Promise.all( [
    34         requestUtils.activateTheme( 'twentytwentyone' ),
    35     ] );
     33    await Promise.all( [ requestUtils.activateTheme( 'twentytwentyone' ) ] );
    3634
    3735    await requestContext.dispose();
  • trunk/tests/performance/config/performance-reporter.js

    r56926 r58076  
    22 * External dependencies
    33 */
    4 import { join, dirname, basename } from 'node:path';
    5 import { writeFileSync } from 'node:fs';
    6 
    7 /**
    8  * Internal dependencies
    9  */
    10 import { getResultsFilename } from '../utils';
     4import { join } from 'node:path';
     5import { writeFileSync, existsSync, mkdirSync } from 'node:fs';
    116
    127/**
     
    1510class PerformanceReporter {
    1611    /**
     12     *
     13     * @type {Record<string,{title: string; results: Record< string, number[] >[];}>}
     14     */
     15    allResults = {};
     16
     17    /**
     18     * Called after a test has been finished in the worker process.
     19     *
     20     * Used to add test results to the final summary of all tests.
    1721     *
    1822     * @param {import('@playwright/test/reporter').TestCase} test
     
    2529
    2630        if ( performanceResults?.body ) {
    27             writeFileSync(
    28                 join(
    29                     dirname( test.location.file ),
    30                     getResultsFilename( basename( test.location.file, '.js' ) )
    31                 ),
    32                 performanceResults.body.toString( 'utf-8' )
     31            // 0 = empty, 1 = browser, 2 = file name, 3 = test suite name, 4 = test name.
     32            const titlePath = test.titlePath();
     33            const title = `${ titlePath[ 3 ] } › ${ titlePath[ 4 ] }`;
     34
     35            // results is an array in case repeatEach is > 1.
     36
     37            this.allResults[ title ] ??= {
     38                file: test.location.file, // Unused, but useful for debugging.
     39                results: [],
     40            };
     41
     42            this.allResults[ title ].results.push(
     43                JSON.parse( performanceResults.body.toString( 'utf-8' ) )
    3344            );
    3445        }
     46    }
     47
     48    /**
     49     * Called after all tests have been run, or testing has been interrupted.
     50     *
     51     * Writes all raw numbers to a file for further processing,
     52     * for example to compare with a previous run.
     53     *
     54     * @param {import('@playwright/test/reporter').FullResult} result
     55     */
     56    onEnd( result ) {
     57        const summary = [];
     58
     59        for ( const [ title, { file, results } ] of Object.entries(
     60            this.allResults
     61        ) ) {
     62            summary.push( {
     63                file,
     64                title,
     65                results,
     66            } );
     67        }
     68
     69        if ( ! existsSync( process.env.WP_ARTIFACTS_PATH ) ) {
     70            mkdirSync( process.env.WP_ARTIFACTS_PATH );
     71        }
     72
     73        const prefix = process.env.TEST_RESULTS_PREFIX;
     74        const fileNamePrefix = prefix ? `${ prefix }-` : '';
     75
     76        writeFileSync(
     77            join(
     78                process.env.WP_ARTIFACTS_PATH,
     79                `${ fileNamePrefix }performance-results.json`
     80            ),
     81            JSON.stringify( summary, null, 2 )
     82        );
    3583    }
    3684}
  • trunk/tests/performance/log-results.js

    r57083 r58076  
    11#!/usr/bin/env node
     2
     3/*
     4 * Get the test results and format them in the way required by the API.
     5 *
     6 * Contains some backward compatibility logic for the original test suite format,
     7 * see #59900 for details.
     8 */
    29
    310/**
    411 * External dependencies.
    512 */
    6 const fs = require( 'fs' );
    7 const path = require( 'path' );
    813const https = require( 'https' );
    9 const [ token, branch, hash, baseHash, timestamp, host ] = process.argv.slice( 2 );
    10 const { median } = require( './utils' );
     14const [ token, branch, hash, baseHash, timestamp, host ] =
     15    process.argv.slice( 2 );
     16const { median, parseFile, accumulateValues } = require( './utils' );
    1117
    12 // The list of test suites to log.
    13 const testSuites = [
    14     'admin',
    15     'admin-l10n',
    16     'home-block-theme',
    17     'home-block-theme-l10n',
    18     'home-classic-theme',
    19     'home-classic-theme-l10n',
    20 ];
    21 
    22 // A list of results to parse based on test suites.
    23 const testResults = testSuites.map(( key ) => ({
    24     key,
    25     file: `${ key }.test.results.json`,
    26 }));
    27 
    28 // A list of base results to parse based on test suites.
    29 const baseResults = testSuites.map(( key ) => ({
    30     key,
    31     file: `base-${ key }.test.results.json`,
    32 }));
     18const testSuiteMap = {
     19    'Admin › Locale: en_US': 'admin',
     20    'Admin › Locale: de_DE': 'admin-l10n',
     21    'Front End › Theme: twentytwentyone, Locale: en_US': 'home-classic-theme',
     22    'Front End › Theme: twentytwentyone, Locale: de_DE':
     23        'home-classic-theme-l10n',
     24    'Front End › Theme: twentytwentythree, Locale: en_US': 'home-block-theme',
     25    'Front End › Theme: twentytwentythree, Locale: de_DE':
     26        'home-block-theme-l10n',
     27};
    3328
    3429/**
    35  * Parse test files into JSON objects.
    36  *
    37  * @param {string} fileName The name of the file.
    38  * @returns An array of parsed objects from each file.
     30 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
    3931 */
    40 const parseFile = ( fileName ) => (
    41     JSON.parse(
    42         fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
    43     )
    44 );
     32const afterStats = parseFile( 'performance-results.json' );
     33
     34/**
     35 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
     36 */
     37const baseStats = parseFile( 'base-performance-results.json' );
     38
     39/**
     40 * @type {Record<string, number>}
     41 */
     42const metrics = {};
     43/**
     44 * @type {Record<string, number>}
     45 */
     46const baseMetrics = {};
     47
     48for ( const { title, results } of afterStats ) {
     49    const testSuiteName = testSuiteMap[ title ];
     50    if ( ! testSuiteName ) {
     51        continue;
     52    }
     53
     54    const baseStat = baseStats.find( ( s ) => s.title === title );
     55
     56    const currResults = accumulateValues( results );
     57    const baseResults = accumulateValues( baseStat.results );
     58
     59    for ( const [ metric, values ] of Object.entries( currResults ) ) {
     60        metrics[ `${ testSuiteName }-${ metric }` ] = median( values );
     61    }
     62
     63    for ( const [ metric, values ] of Object.entries( baseResults ) ) {
     64        baseMetrics[ `${ testSuiteName }-${ metric }` ] = median( values );
     65    }
     66}
     67
     68process.exit( 0 );
    4569
    4670/**
     
    4872 *
    4973 * @param {Object[]} results A list of results to format.
    50  * @return {Object[]} Metrics.
     74 * @return {Object} Metrics.
    5175 */
    5276const formatResults = ( results ) => {
    53     return results.reduce(
    54         ( result, { key, file } ) => {
    55             return {
    56                 ...result,
    57                 ...Object.fromEntries(
    58                     Object.entries(
    59                         parseFile( file ) ?? {}
    60                     ).map( ( [ metric, value ] ) => [
     77    return results.reduce( ( result, { key, file } ) => {
     78        return {
     79            ...result,
     80            ...Object.fromEntries(
     81                Object.entries( parseFile( file ) ?? {} ).map(
     82                    ( [ metric, value ] ) => [
    6183                        key + '-' + metric,
    62                         median ( value ),
    63                     ] )
    64                 ),
    65             };
    66         },
    67         {}
    68     );
     84                        median( value ),
     85                    ]
     86                )
     87            ),
     88        };
     89    }, {} );
    6990};
    7091
  • trunk/tests/performance/playwright.config.js

    r57083 r58076  
    2424    workers: 1,
    2525    retries: 0,
     26    repeatEach: 2,
    2627    timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 600_000, // Defaults to 10 minutes.
    2728    // Don't report slow test "files", as we will be running our tests in serial.
    2829    reportSlowTests: null,
     30    preserveOutput: 'never',
    2931    webServer: {
    3032        ...baseConfig.webServer,
     
    3840
    3941export default config;
    40 
  • trunk/tests/performance/specs/admin.test.js

    r57083 r58076  
    1313};
    1414
     15const locales = [ 'en_US', 'de_DE' ];
     16
    1517test.describe( 'Admin', () => {
    16     test.beforeAll( async ( { requestUtils } ) => {
    17         await requestUtils.activateTheme( 'twentytwentyone' );
    18     } );
     18    for ( const locale of locales ) {
     19        test.describe( `Locale: ${ locale }`, () => {
     20            test.beforeAll( async ( { requestUtils } ) => {
     21                await requestUtils.activateTheme( 'twentytwentyone' );
     22                await requestUtils.updateSiteSettings( {
     23                    language: 'en_US' === locale ? '' : locale,
     24                } );
     25            } );
    1926
    20     test.afterAll( async ( {}, testInfo ) => {
    21         await testInfo.attach( 'results', {
    22             body: JSON.stringify( results, null, 2 ),
    23             contentType: 'application/json',
    24         } );
    25     } );
     27            test.afterAll( async ( { requestUtils }, testInfo ) => {
     28                await testInfo.attach( 'results', {
     29                    body: JSON.stringify( results, null, 2 ),
     30                    contentType: 'application/json',
     31                } );
    2632
    27     const iterations = Number( process.env.TEST_RUNS );
    28     for ( let i = 1; i <= iterations; i++ ) {
    29         test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
    30             admin,
    31             metrics,
    32         } ) => {
    33             await admin.visitAdminPage( '/' );
     33                await requestUtils.updateSiteSettings( {
     34                    language: '',
     35                } );
    3436
    35             const serverTiming = await metrics.getServerTiming();
     37                results.timeToFirstByte = [];
     38            } );
    3639
    37             for ( const [ key, value ] of Object.entries( serverTiming ) ) {
    38                 results[ camelCaseDashes( key ) ] ??= [];
    39                 results[ camelCaseDashes( key ) ].push( value );
     40            test.afterAll( async ( {}, testInfo ) => {
     41                await testInfo.attach( 'results', {
     42                    body: JSON.stringify( results, null, 2 ),
     43                    contentType: 'application/json',
     44                } );
     45            } );
     46
     47            const iterations = Number( process.env.TEST_RUNS );
     48            for ( let i = 1; i <= iterations; i++ ) {
     49                test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
     50                    admin,
     51                    metrics,
     52                } ) => {
     53                    await admin.visitAdminPage( '/' );
     54
     55                    const serverTiming = await metrics.getServerTiming();
     56
     57                    for ( const [ key, value ] of Object.entries(
     58                        serverTiming
     59                    ) ) {
     60                        results[ camelCaseDashes( key ) ] ??= [];
     61                        results[ camelCaseDashes( key ) ].push( value );
     62                    }
     63
     64                    const ttfb = await metrics.getTimeToFirstByte();
     65                    results.timeToFirstByte.push( ttfb );
     66                } );
    4067            }
    41 
    42             const ttfb = await metrics.getTimeToFirstByte();
    43             results.timeToFirstByte.push( ttfb );
    4468        } );
    4569    }
  • trunk/tests/performance/utils.js

    r56928 r58076  
     1/**
     2 * External dependencies.
     3 */
     4const { readFileSync, existsSync } = require( 'node:fs' );
     5const { join } = require( 'node:path' );
     6
     7process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
     8
     9/**
     10 * Parse test files into JSON objects.
     11 *
     12 * @param {string} fileName The name of the file.
     13 * @return {Array<{file: string, title: string, results: Record<string,number[]>[]}>} Parsed object.
     14 */
     15function parseFile( fileName ) {
     16    const file = join( process.env.WP_ARTIFACTS_PATH, fileName );
     17    if ( ! existsSync( file ) ) {
     18        return [];
     19    }
     20
     21    return JSON.parse( readFileSync( file, 'utf8' ) );
     22}
     23
    124/**
    225 * Computes the median number from an array numbers.
     
    1437}
    1538
    16 /**
    17  * Gets the result file name.
    18  *
    19  * @param {string} fileName File name.
    20  *
    21  * @return {string} Result file name.
    22  */
    23 function getResultsFilename( fileName ) {
    24     const prefix = process.env.TEST_RESULTS_PREFIX;
    25     const fileNamePrefix = prefix ? `${ prefix }-` : '';
    26     return `${fileNamePrefix + fileName}.results.json`;
    27 }
    28 
    2939function camelCaseDashes( str ) {
    30     return str.replace( /-([a-z])/g, function( g ) {
     40    return str.replace( /-([a-z])/g, function ( g ) {
    3141        return g[ 1 ].toUpperCase();
    3242    } );
    3343}
    3444
     45/**
     46 * Formats an array of objects as a Markdown table.
     47 *
     48 * For example, this array:
     49 *
     50 * [
     51 *  {
     52 *      foo: 123,
     53 *      bar: 456,
     54 *      baz: 'Yes',
     55 *  },
     56 *  {
     57 *      foo: 777,
     58 *      bar: 999,
     59 *      baz: 'No',
     60 *  }
     61 * ]
     62 *
     63 * Will result in the following table:
     64 *
     65 * | foo | bar | baz |
     66 * |-----|-----|-----|
     67 * | 123 | 456 | Yes |
     68 * | 777 | 999 | No  |
     69 *
     70 * @param {Array<Object>} rows Table rows.
     71 * @returns {string} Markdown table content.
     72 */
     73function formatAsMarkdownTable( rows ) {
     74    let result = '';
     75
     76    if ( ! rows.length ) {
     77        return result;
     78    }
     79
     80    const headers = Object.keys( rows[ 0 ] );
     81    for ( const header of headers ) {
     82        result += `| ${ header } `;
     83    }
     84    result += '|\n';
     85    for ( const header of headers ) {
     86        result += '| ------ ';
     87    }
     88    result += '|\n';
     89
     90    for ( const row of rows ) {
     91        for ( const value of Object.values( row ) ) {
     92            result += `| ${ value } `;
     93        }
     94        result += '|\n';
     95    }
     96
     97    return result;
     98}
     99
     100/**
     101 * Nicely formats a given value.
     102 *
     103 * @param {string} metric Metric.
     104 * @param {number} value
     105 */
     106function formatValue( metric, value ) {
     107    if ( null === value ) {
     108        return 'N/A';
     109    }
     110
     111    if ( 'wpMemoryUsage' === metric ) {
     112        return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
     113    }
     114
     115    if ( 'wpExtObjCache' === metric ) {
     116        return 1 === value ? 'yes' : 'no';
     117    }
     118
     119    if ( 'wpDbQueries' === metric ) {
     120        return value;
     121    }
     122
     123    return `${ value.toFixed( 2 ) } ms`;
     124}
     125
     126/**
     127 * Returns a Markdown link to a Git commit on the current GitHub repository.
     128 *
     129 * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e`
     130 * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a].
     131 *
     132 * @param {string} sha Commit SHA.
     133 * @return string Link
     134 */
     135function linkToSha( sha ) {
     136    const repoName =
     137        process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop';
     138
     139    return `[${ sha.slice(
     140        0,
     141        7
     142    ) }](https://github.com/${ repoName }/commit/${ sha })`;
     143}
     144
     145function standardDeviation( array = [] ) {
     146    if ( ! array.length ) {
     147        return 0;
     148    }
     149
     150    const mean = array.reduce( ( a, b ) => a + b ) / array.length;
     151    return Math.sqrt(
     152        array
     153            .map( ( x ) => Math.pow( x - mean, 2 ) )
     154            .reduce( ( a, b ) => a + b ) / array.length
     155    );
     156}
     157
     158function medianAbsoluteDeviation( array = [] ) {
     159    if ( ! array.length ) {
     160        return 0;
     161    }
     162
     163    const med = median( array );
     164    return median( array.map( ( a ) => Math.abs( a - med ) ) );
     165}
     166
     167/**
     168 *
     169 * @param {Array<Record<string, number[]>>} results
     170 * @returns {Record<string, number[]>}
     171 */
     172function accumulateValues( results ) {
     173    return results.reduce( ( acc, result ) => {
     174        for ( const [ metric, values ] of Object.entries( result ) ) {
     175            acc[ metric ] = acc[ metric ] ?? [];
     176            acc[ metric ].push( ...values );
     177        }
     178        return acc;
     179    }, {} );
     180}
     181
    35182module.exports = {
     183    parseFile,
    36184    median,
    37     getResultsFilename,
    38185    camelCaseDashes,
     186    formatAsMarkdownTable,
     187    formatValue,
     188    linkToSha,
     189    standardDeviation,
     190    medianAbsoluteDeviation,
     191    accumulateValues,
    39192};
  • trunk/tests/performance/wp-content/mu-plugins/server-timing.php

    r57083 r58076  
    55    static function ( $template ) {
    66
    7         global $timestart;
     7        global $timestart, $wpdb;
    88
    99        $server_timing_values = array();
     
    1616        add_action(
    1717            'shutdown',
    18             static function () use ( $server_timing_values, $template_start ) {
    19 
    20                 global $timestart;
    21 
     18            static function () use ( $server_timing_values, $template_start, $wpdb ) {
    2219                $output = ob_get_clean();
    2320
     
    3128                 * This is a nice little trick as it allows to easily get this information in JS.
    3229                 */
    33                 $server_timing_values['memory-usage'] = memory_get_usage();
     30                $server_timing_values['memory-usage']  = memory_get_usage();
     31                $server_timing_values['db-queries']    = $wpdb->num_queries;
     32                $server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0;
    3433
    3534                $header_values = array();
     
    5150    PHP_INT_MAX
    5251);
     52
     53add_action(
     54    'admin_init',
     55    static function () {
     56        global $timestart, $wpdb;
     57
     58        ob_start();
     59
     60        add_action(
     61            'shutdown',
     62            static function () use ( $wpdb, $timestart ) {
     63                $output = ob_get_clean();
     64
     65                $server_timing_values = array();
     66
     67                $server_timing_values['total'] = microtime( true ) - $timestart;
     68
     69                /*
     70                 * While values passed via Server-Timing are intended to be durations,
     71                 * any numeric value can actually be passed.
     72                 * This is a nice little trick as it allows to easily get this information in JS.
     73                 */
     74                $server_timing_values['memory-usage']  = memory_get_usage();
     75                $server_timing_values['db-queries']    = $wpdb->num_queries;
     76                $server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0;
     77
     78                $header_values = array();
     79                foreach ( $server_timing_values as $slug => $value ) {
     80                    if ( is_float( $value ) ) {
     81                        $value = round( $value * 1000.0, 2 );
     82                    }
     83                    $header_values[] = sprintf( 'wp-%1$s;dur=%2$s', $slug, $value );
     84                }
     85                header( 'Server-Timing: ' . implode( ', ', $header_values ) );
     86
     87                echo $output;
     88            },
     89            PHP_INT_MIN
     90        );
     91    },
     92    PHP_INT_MAX
     93);
Note: See TracChangeset for help on using the changeset viewer.