Add the check-sarif action

aeisenberg · aeisenberg · commit bcb7fad5b36d · 2022-06-14T11:55:10.000-07:00
Allows us to analyze and then check that certain queries were included
in the analysis and others were not.
diff --git a/.github/check-sarif/action.yml b/.github/check-sarif/action.yml
@@ -0,0 +1,20 @@
+name: Check SARIF
+description: Checks a SARIF file to see if certain queries were run and others were not run.
+inputs:
+  sarif-file:
+    required: true
+    description: The sarif file to check
+
+  queries-run:
+    required: true
+    description: |
+      Comma separated list of query ids that should be included in this SARIF file.
+
+  queries-not-run:
+    required: true
+    description: |
+      Comma separated list of query ids that should NOT be included in this SARIF file.
+
+runs:
+  using: node12
+  main: index.js
diff --git a/.github/check-sarif/index.js b/.github/check-sarif/index.js
@@ -0,0 +1,52 @@
+'use strict'
+
+const core = require('@actions/core');
+const fs = require('fs')
+
+const sarif = JSON.parse(fs.readFileSync(core.getInput('sarif-file'), 'utf8'))
+const rules = sarif.runs[0].tool.extensions.flatMap(ext => ext.rules || [])
+
+// Expected Queries
+const expectedQueriesRun = getInput('queries-run')
+const queriesThatShouldHaveRunButDidnt = expectedQueriesRun.reduce((acc, queryId) => {
+  if (!rules.some(rule => rule.id === queryId)) {
+    acc.push(queryId)
+  }
+  return acc
+}, []);
+
+if (queriesThatShouldHaveRunButDidnt.length > 0) {
+  core.setFailed(`The following queries were expected to run but did not: ${queriesThatShouldHaveRunButDidnt.join(', ')}`)
+}
+
+// Unexpected Queries
+const expectedQueriesNotRun = getInput('queries-not-run')
+
+const queriesThatShouldNotHaveRunButDid = expectedQueriesNotRun.reduce((acc, queryId) => {
+  if (rules.some(rule => rule.id === queryId)) {
+    acc.push(queryId)
+  }
+  return acc
+}, []);
+
+if (queriesThatShouldNotHaveRunButDid.length > 0) {
+  core.setFailed(`The following queries were NOT expected to have run but did: ${queriesThatShouldNotHaveRunButDid.join(', ')}`)
+}
+
+
+core.startGroup('All queries run')
+rules.forEach(rule => {
+  core.info(`${rule.id}: ${(rule.properties && rule.properties.name) || rule.name}`)
+})
+core.endGroup()
+
+core.startGroup('Full SARIF')
+core.info(JSON.stringify(sarif, null, 2))
+core.endGroup()
+
+function getInput(name) {
+  return core.getInput(name)
+    .split(',')
+    .map(q => q.trim())
+    .filter(q => q.length > 0)
+}
diff --git a/.github/workflows/expected-queries-runs.yml b/.github/workflows/expected-queries-runs.yml
@@ -0,0 +1,48 @@
+name: Expected queries runs
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+on:
+  push:
+    branches:
+    - main
+    - releases/v1
+    - releases/v2
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+
+jobs:
+  expected-queries:
+    timeout-minutes: 45
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v3
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: latest
+    - uses: ./../action/init
+      with:
+        languages: javascript
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - uses: ./../action/analyze
+      with:
+        output: ${{ runner.temp }}/results
+        upload-database: false
+        upload: false
+      env:
+        TEST_MODE: true
+
+    - name: Check Sarif
+      uses: ./../action/.github/check-sarif
+      with:
+        sarif-file:  ${{ runner.temp }}/results/javascript.sarif
+        queries-run: js/incomplete-hostname-regexp,js/path-injection
+        queries-not-run: foo,bar
