Make CI collapse previous PR comments with profiling benchmark deltas (#1974)

CI change again CI collapse by default
2026-09-15 14:18:04 +08:00 · 2024-09-13 22:54:33 -07:00
parent 514582fd8d
commit 426f3b2cb4
5 changed files with 49 additions and 14 deletions
--- a/.github/workflows/comment-profiling-changes.yaml
+++ b/.github/workflows/comment-profiling-changes.yaml
@@ -0,0 +1,168 @@
+name: Profiling Changes
+
+on:
+  pull_request:
+    branches: [master]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  profile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+
+      - name: Install Valgrind
+        run: |
+          sudo apt update
+          sudo apt install -y valgrind
+
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install iai-callgrind
+        run: |
+          cargo install iai-callgrind-runner@0.12.3
+
+      - name: Checkout master branch
+        run: |
+          git fetch origin master:master
+          git checkout master
+
+      - name: Run baseline benchmarks
+        run: |
+          cargo bench --bench compile_demo_art --features=iai -- --save-baseline=master
+
+      - name: Checkout PR branch
+        run: |
+          git checkout ${{ github.event.pull_request.head.sha }}
+
+      - name: Run PR benchmarks
+        id: benchmark
+        run: |
+          BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json  | jq -sc | sed 's/\\"//g')
+          echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_OUTPUT
+          echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Make old comments collapsed by default
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const { data: comments } = await github.rest.issues.listComments({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            });
+
+            const botComments = comments.filter((comment) =>
+              comment.user.type === 'Bot' && comment.body.includes('Performance Benchmark Results') && comment.body.includes('<details open>')
+            );
+
+            for (const comment of botComments) {
+              // Edit the comment to remove the "open" attribute from the <details> tag
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: comment.id,
+                body: comment.body.replace('<details open>', '<details>')
+              });
+            }
+
+      - name: Comment PR
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const benchmarkOutput = JSON.parse(`${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}`);
+            let significantChanges = false;
+            let commentBody = "";
+
+            function formatNumber(num) {
+              return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+            }
+
+            function formatPercentage(pct) {
+              const sign = pct >= 0 ? '+' : '';
+              return `${sign}${pct.toFixed(2)}%`;
+            }
+
+            function padRight(str, len) {
+              return str.padEnd(len);
+            }
+
+            function padLeft(str, len) {
+              return str.padStart(len);
+            }
+
+            for (const benchmark of benchmarkOutput) {
+              if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) {
+                const summary = benchmark.callgrind_summary.summaries[0];
+                const irDiff = summary.events.Ir;
+                
+                if (irDiff.diff_pct !== null) {
+                  const changePercentage = formatPercentage(irDiff.diff_pct);
+                  const color = irDiff.diff_pct > 0 ? "red" : "lime";
+                  
+                  commentBody += "---\n\n";
+                  commentBody += `${benchmark.module_path} ${benchmark.id}:${benchmark.details}\n`;
+                  commentBody += `Instructions: \`${formatNumber(irDiff.old)}\` (master) -> \`${formatNumber(irDiff.new)}\` (HEAD) : `;
+                  commentBody += `$$\\color{${color}}${changePercentage.replace("%", "\\\\%")}$$\n\n`;
+                  
+                  commentBody += "<details>\n<summary>Detailed metrics</summary>\n\n```\n";
+                  commentBody += `Baselines:                master|       HEAD\n`;
+                  
+                  for (const [eventKind, costsDiff] of Object.entries(summary.events)) {
+                    if (costsDiff.diff_pct !== null) {
+                      const changePercentage = formatPercentage(costsDiff.diff_pct);
+                      const line = `${padRight(eventKind, 20)} ${padLeft(formatNumber(costsDiff.old), 11)}|${padLeft(formatNumber(costsDiff.new), 11)} ${padLeft(changePercentage, 15)}`;
+                      commentBody += `${line}\n`;
+                    }
+                  }
+                  
+                  commentBody += "```\n</details>\n\n";
+                  
+                  if (Math.abs(irDiff.diff_pct) > 5) {
+                    significantChanges = true;
+                  }
+                }
+              }
+            }
+
+            const output = `
+            <details open>
+
+            <summary>Performance Benchmark Results</summary>
+
+            ${commentBody}
+
+            </details>
+            `;
+
+            if (significantChanges) {
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: output
+              });
+            } else {
+              console.log("No significant performance changes detected. Skipping comment.");
+              console.log(output);
+            }