Changeset 58076 for trunk/.github/workflows/performance.yml

Timestamp:

05/02/2024 01:57:49 PM (14 months ago)

Author:

swissspidy

Message:

Build/Test Tools: Overhaul performance tests to improve stability and cover more scenarios.

Simplifies the tests setup by leveraging a test matrix, improving maintenance and making it much easier to test more scenarios. With this change, tests are now also run with an external object cache (Memcached). Additional information such as memory usage and the number of database queries is now collected as well.

Improves test setup and cleanup by disabling external HTTP requests and cron for the tests, as well as deleting expired transients and flushing the cache in-between. This should aid the test stability.

When testing the previous commit / target branch, this now leverages the already built artifact from the build process workflow. Raw test results are now also uploaded as artifacts to aid debugging.

Props swissspidy, adamsilverstein, joemcgill, mukesh27, desrosj, youknowriad, flixos90.
Fixes #59900

File:

• trunk/.github/workflows/performance.yml (modified) (11 diffs)

trunk/.github/workflows/performance.yml

@@ -67 +67 @@
   # - Install WordPress Importer plugin.
   # - Import mock data.
+  # - Deactivate WordPress Importer plugin.
   # - Update permalink structure.
+  # - Install additional languages.
+  # - Disable external HTTP requests.
+  # - Disable cron.
+  # - List defined constants.
   # - Install MU plugin.
   # - Run performance tests (current commit).
-  # - Print performance tests results.
-  # - Check out target commit (target branch or previous commit).
-  # - Switch Node.js versions if necessary.
-  # - Install npm dependencies.
-  # - Build WordPress.
+  # - Download previous build artifact (target branch or previous commit).
+  # - Download artifact.
+  # - Unzip the build.
   # - Run any database upgrades.
+  # - Flush cache.
+  # - Delete expired transients.
   # - Run performance tests (previous/target commit).
-  # - Print target performance tests results.
-  # - Reset to original commit.
-  # - Switch Node.js versions if necessary.
-  # - Install npm dependencies.
   # - Set the environment to the baseline version.
   # - Run any database upgrades.
+  # - Flush cache.
+  # - Delete expired transients.
   # - Run baseline performance tests.
-  # - Print baseline performance tests results.
-  # - Compare results with base.
+  # - Archive artifacts.
+  # - Compare results.
   # - Add workflow summary.
   # - Set the base sha.
@@ -91 +94 @@
   # - Publish performance results.
   # - Ensure version-controlled files are not modified or deleted.
-  # - Dispatch workflow run.
   performance:
-    name: Run performance tests
+    name: Run performance tests ${{ matrix.memcached && '(with memcached)' || '' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
     if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) && ! contains( github.event.before, '00000000' ) }}
-
+    strategy:
+      fail-fast: false
+      matrix:
+        memcached: [ true, false ]
+    env:
+      LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
     steps:
       - name: Configure environment variables
@@ -128 +135 @@
 
       - name: Install Playwright browsers
-        run: npx playwright install --with-deps
+        run: npx playwright install --with-deps chromium
 
       - name: Build WordPress
@@ -134 +141 @@
 
       - name: Start Docker environment
-        run: |
-          npm run env:start
+        run: npm run env:start
+
+      - name: Install object cache drop-in
+        if: ${{ matrix.memcached }}
+        run: cp src/wp-content/object-cache.php build/wp-content/object-cache.php
 
       - name: Log running Docker containers
@@ -161 +171 @@
           rm themeunittestdata.wordpress.xml
 
+      - name: Deactivate WordPress Importer plugin
+        run: npm run env:cli -- plugin deactivate wordpress-importer --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Update permalink structure
-        run: |
-          npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
+        run: npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
 
       - name: Install additional languages
@@ -171 +183 @@
           npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
 
+      # Prevent background update checks from impacting test stability.
+      - name: Disable external HTTP requests
+        run: npm run env:cli -- config set WP_HTTP_BLOCK_EXTERNAL true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
+
+      # Prevent background tasks from impacting test stability.
+      - name: Disable cron
+        run: npm run env:cli -- config set DISABLE_WP_CRON true --raw --type=constant --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: List defined constants
+        run: npm run env:cli -- config list --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Install MU plugin
         run: |
@@ -179 +202 @@
         run: npm run test:performance
 
-      - name: Print performance tests results
-        run: node ./tests/performance/results.js
-
-      - name: Check out target commit (target branch or previous commit)
-        run: |
-          if [[ -z "$TARGET_REF" ]]; then
-            git fetch -n origin $TARGET_SHA
-          else
-            git fetch -n origin $TARGET_REF
-          fi
-          git reset --hard $TARGET_SHA
-
-      - name: Set up Node.js
-        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
-        with:
-          node-version-file: '.nvmrc'
-          cache: npm
-
-      - name: Install npm dependencies
-        run: npm ci
-
-      - name: Build WordPress
-        run: npm run build
+      - name: Download previous build artifact (target branch or previous commit)
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        id: get-previous-build
+        with:
+          script: |
+            const artifacts = await github.rest.actions.listArtifactsForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'wordpress-build-' + process.env.TARGET_SHA,
+            });
+
+            const matchArtifact = artifacts.data.artifacts[0];
+
+            if ( ! matchArtifact ) {
+              core.setFailed( 'No artifact found!' );
+              return false;
+            }
+
+            const download = await github.rest.actions.downloadArtifact( {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: matchArtifact.id,
+              archive_format: 'zip',
+            } );
+
+            const fs = require( 'fs' );
+            fs.writeFileSync( '${{ github.workspace }}/before.zip', Buffer.from( download.data ) )
+
+            return true;
+
+      - name: Unzip the build
+        if: ${{ steps.get-previous-build.outputs.result }}
+        run: |
+          unzip ${{ github.workspace }}/before.zip
+          unzip -o ${{ github.workspace }}/wordpress.zip
 
       - name: Run any database upgrades
+        if: ${{ steps.get-previous-build.outputs.result }}
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
-      - name: Run target performance tests (base/previous commit)
+      - name: Flush cache
+        if: ${{ steps.get-previous-build.outputs.result }}
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Delete expired transients
+        if: ${{ steps.get-previous-build.outputs.result }}
+        run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Run target performance tests (previous/target commit)
+        if: ${{ steps.get-previous-build.outputs.result }}
         env:
           TEST_RESULTS_PREFIX: before
         run: npm run test:performance
 
-      - name: Print target performance tests results
-        env:
-          TEST_RESULTS_PREFIX: before
-        run: node ./tests/performance/results.js
-
-      - name: Reset to original commit
-        run: git reset --hard $GITHUB_SHA
-
-      - name: Set up Node.js
-        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
-        with:
-          node-version-file: '.nvmrc'
-          cache: npm
-
-      - name: Install npm dependencies
-        run: npm ci
-
       - name: Set the environment to the baseline version
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: |
           npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
@@ -234 +263 @@
 
       - name: Run any database upgrades
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Flush cache
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Delete expired transients
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Run baseline performance tests
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
         env:
           TEST_RESULTS_PREFIX: base
         run: npm run test:performance
 
-      - name: Print baseline performance tests results
-        env:
-          TEST_RESULTS_PREFIX: base
-        run: node ./tests/performance/results.js
-
-      - name: Compare results with base
+      - name: Archive artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        if: always()
+        with:
+          name: performance-artifacts${{ matrix.memcached && '-memcached' || '' }}-${{ github.run_id }}
+          path: artifacts
+          if-no-files-found: ignore
+
+      - name: Compare results
         run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
 
@@ -254 +296 @@
       - name: Set the base sha
         # Only needed when publishing results.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         id: base-sha
@@ -265 +307 @@
       - name: Set commit details
         # Only needed when publishing results.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         id: commit-timestamp
@@ -276 +318 @@
       - name: Publish performance results
         # Only publish results on pushes to trunk.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! matrix.memcached }}
         env:
             BASE_SHA: ${{ steps.base-sha.outputs.result }}