Include graph runtime benchmarks in CI perf regression runs (#2780)

* Include graph runtime benchmarks in ci regression run

* Update benchmarking workflow

* Add render slowdown

* Fix baseline cache

* Remove benchmark compilation validation run

* Include render node in runtime benchmarks

* Collapse sections without changes

* Readd rulers between the sections

* Add review suggestions

* Rulers rule

* Fix whitespace

.github/workflows/comment-profiling-changes.yaml

@@ -10,31 +10,33 @@ jobs:
   profile:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
 
       - name: Install Valgrind
         run: |
           sudo apt update
           sudo apt install -y valgrind
 
-      - name: Cache dependencies
-        uses: actions/cache@v3
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          # Cache on Cargo.lock file
+          cache-on-failure: true
+
+      - name: Cache iai-callgrind binary
+        id: cache-iai
+        uses: actions/cache@v4
+        with:
+          path: ~/.cargo/bin/iai-callgrind-runner
+          key: ${{ runner.os }}-iai-callgrind-runner-0.12.3
 
       - name: Install iai-callgrind
+        if: steps.cache-iai.outputs.cache-hit != 'true'
         run: |
           cargo install iai-callgrind-runner@0.12.3
 
@@ -43,9 +45,29 @@ jobs:
           git fetch origin master:master
           git checkout master
 
+      - name: Get master commit SHA
+        id: master-sha
+        run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Cache benchmark baselines
+        id: cache-benchmark-baselines
+        uses: actions/cache@v4
+        with:
+          path: target/iai
+          key: ${{ runner.os }}-benchmark-baselines-master-${{ steps.master-sha.outputs.sha }}
+          restore-keys: |
+            ${{ runner.os }}-benchmark-baselines-master-
+
       - name: Run baseline benchmarks
+        if: steps.cache-benchmark-baselines.outputs.cache-hit != 'true'
         run: |
+          # Compile benchmarks
           cargo bench --bench compile_demo_art_iai -- --save-baseline=master
+          
+          # Runtime benchmarks
+          cargo bench --bench update_executor_iai -- --save-baseline=master
+          cargo bench --bench run_once_iai -- --save-baseline=master
+          cargo bench --bench run_cached_iai -- --save-baseline=master
 
       - name: Checkout PR branch
         run: |
@@ -54,13 +76,33 @@ jobs:
       - name: Run PR benchmarks
         id: benchmark
         run: |
-          BENCH_OUTPUT=$(cargo bench --bench compile_demo_art_iai -- --baseline=master --output-format=json  | jq -sc | sed 's/\\"//g')
-          echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_OUTPUT
-          echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT
+          # Compile benchmarks
+          COMPILE_OUTPUT=$(cargo bench --bench compile_demo_art_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')
+          
+          # Runtime benchmarks
+          UPDATE_OUTPUT=$(cargo bench --bench update_executor_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')
+          RUN_ONCE_OUTPUT=$(cargo bench --bench run_once_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')
+          RUN_CACHED_OUTPUT=$(cargo bench --bench run_cached_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')
+          
+          # Store outputs
+          echo "COMPILE_OUTPUT<<EOF" >> $GITHUB_OUTPUT
+          echo "$COMPILE_OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          
+          echo "UPDATE_OUTPUT<<EOF" >> $GITHUB_OUTPUT
+          echo "$UPDATE_OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          
+          echo "RUN_ONCE_OUTPUT<<EOF" >> $GITHUB_OUTPUT
+          echo "$RUN_ONCE_OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          
+          echo "RUN_CACHED_OUTPUT<<EOF" >> $GITHUB_OUTPUT
+          echo "$RUN_CACHED_OUTPUT" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Make old comments collapsed by default
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
@@ -85,11 +127,15 @@ jobs:
             }
 
       - name: Comment PR
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
-            const benchmarkOutput = JSON.parse(`${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}`);
+            const compileOutput = JSON.parse(`${{ steps.benchmark.outputs.COMPILE_OUTPUT }}`);
+            const updateOutput = JSON.parse(`${{ steps.benchmark.outputs.UPDATE_OUTPUT }}`);
+            const runOnceOutput = JSON.parse(`${{ steps.benchmark.outputs.RUN_ONCE_OUTPUT }}`);
+            const runCachedOutput = JSON.parse(`${{ steps.benchmark.outputs.RUN_CACHED_OUTPUT }}`);
+            
             let significantChanges = false;
             let commentBody = "";
 
@@ -110,58 +156,97 @@ jobs:
               return str.padStart(len);
             }
 
-            for (const benchmark of benchmarkOutput) {
-              if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) {
-                const summary = benchmark.callgrind_summary.summaries[0];
-                const irDiff = summary.events.Ir;
-                
-                if (irDiff.diff_pct !== null) {
-                  const changePercentage = formatPercentage(irDiff.diff_pct);
-                  const color = irDiff.diff_pct > 0 ? "red" : "lime";
+            function processBenchmarkOutput(benchmarkOutput, sectionTitle, isLast = false) {
+              let sectionBody = "";
+              let hasResults = false;
+              let hasSignificantChanges = false;
+              
+              for (const benchmark of benchmarkOutput) {
+                if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) {
+                  const summary = benchmark.callgrind_summary.summaries[0];
+                  const irDiff = summary.events.Ir;
                   
-                  commentBody += "---\n\n";
-                  commentBody += `${benchmark.module_path} ${benchmark.id}:${benchmark.details}\n`;
-                  commentBody += `Instructions: \`${formatNumber(irDiff.old)}\` (master) -> \`${formatNumber(irDiff.new)}\` (HEAD) : `;
-                  commentBody += `$$\\color{${color}}${changePercentage.replace("%", "\\\\%")}$$\n\n`;
-                  
-                  commentBody += "<details>\n<summary>Detailed metrics</summary>\n\n```\n";
-                  commentBody += `Baselines:                master|       HEAD\n`;
-                  
-                  for (const [eventKind, costsDiff] of Object.entries(summary.events)) {
-                    if (costsDiff.diff_pct !== null) {
-                      const changePercentage = formatPercentage(costsDiff.diff_pct);
-                      const line = `${padRight(eventKind, 20)} ${padLeft(formatNumber(costsDiff.old), 11)}|${padLeft(formatNumber(costsDiff.new), 11)} ${padLeft(changePercentage, 15)}`;
-                      commentBody += `${line}\n`;
+                  if (irDiff.diff_pct !== null) {
+                    hasResults = true;
+                    const changePercentage = formatPercentage(irDiff.diff_pct);
+                    const color = irDiff.diff_pct > 0 ? "red" : "lime";
+                    
+                    sectionBody += `**${benchmark.module_path} ${benchmark.id}:${benchmark.details}**\n`;
+                    sectionBody += `Instructions: \`${formatNumber(irDiff.old)}\` (master) → \`${formatNumber(irDiff.new)}\` (HEAD) : `;
+                    sectionBody += `$$\\color{${color}}${changePercentage.replace("%", "\\\\%")}$$\n\n`;
+                    
+                    sectionBody += "<details>\n<summary>Detailed metrics</summary>\n\n```\n";
+                    sectionBody += `Baselines:                master|       HEAD\n`;
+                    
+                    for (const [eventKind, costsDiff] of Object.entries(summary.events)) {
+                      if (costsDiff.diff_pct !== null) {
+                        const changePercentage = formatPercentage(costsDiff.diff_pct);
+                        const line = `${padRight(eventKind, 20)} ${padLeft(formatNumber(costsDiff.old), 11)}|${padLeft(formatNumber(costsDiff.new), 11)} ${padLeft(changePercentage, 15)}`;
+                        sectionBody += `${line}\n`;
+                      }
+                    }
+                    
+                    sectionBody += "```\n</details>\n\n";
+                    
+                    if (Math.abs(irDiff.diff_pct) > 5) {
+                      significantChanges = true;
+                      hasSignificantChanges = true;
                     }
-                  }
-                  
-                  commentBody += "```\n</details>\n\n";
-                  
-                  if (Math.abs(irDiff.diff_pct) > 5) {
-                    significantChanges = true;
                   }
                 }
               }
+              
+              if (hasResults) {
+                // Wrap section in collapsible details, open only if there are significant changes
+                const openAttribute = hasSignificantChanges ? " open" : "";
+                const ruler = isLast ? "" : "\n\n---";
+                return `<details${openAttribute}>\n<summary><h2>${sectionTitle}</h2></summary>\n\n${sectionBody}${ruler}\n</details>`;
+              }
+              return "";
             }
 
-            const output = `
-            <details open>
+            // Process each benchmark category
+            const sections = [
+              { output: compileOutput, title: "🔧 Graph Compilation" },
+              { output: updateOutput, title: "🔄 Executor Update" },
+              { output: runOnceOutput, title: "🚀 Render: Cold Execution" },
+              { output: runCachedOutput, title: "⚡ Render: Cached Execution" }
+            ];
 
-            <summary>Performance Benchmark Results</summary>
+            // Generate sections and determine which ones have results
+            const generatedSections = sections.map(({ output, title }) => 
+              processBenchmarkOutput(output, title, true) // temporarily mark all as last
+            ).filter(section => section.length > 0);
 
-            ${commentBody}
+            // Re-generate with correct isLast flags
+            let sectionIndex = 0;
+            const finalSections = sections.map(({ output, title }) => {
+              const section = processBenchmarkOutput(output, title, true); // check if it has results
+              if (section.length > 0) {
+                const isLast = sectionIndex === generatedSections.length - 1;
+                sectionIndex++;
+                return processBenchmarkOutput(output, title, isLast);
+              }
+              return "";
+            }).filter(section => section.length > 0);
 
-            </details>
-            `;
+            // Combine all sections
+            commentBody = finalSections.join("\n\n");
 
-            if (significantChanges) {
-              github.rest.issues.createComment({
-                issue_number: context.issue.number,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: output
-              });
+            if (commentBody.length > 0) {
+              const output = `<details open>\n<summary>Performance Benchmark Results</summary>\n\n${commentBody}\n</details>`;
+
+              if (significantChanges) {
+                github.rest.issues.createComment({
+                  issue_number: context.issue.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body: output
+                });
+              } else {
+                console.log("No significant performance changes detected. Skipping comment.");
+                console.log(output);
+              }
             } else {
-              console.log("No significant performance changes detected. Skipping comment.");
-              console.log(output);
+              console.log("No benchmark results to display.");
             }