Support overlay database creation

This commit adds support for creating overlay-base and overlay databases, controlled via the CODEQL_OVERLAY_DATABASE_MODE environment variable.
github · smowton · Mar 19, 2025 · Mar 7, 2025 · Mar 7, 2025 · Mar 7, 2025
commit ff5f0b9efdde997429172d6cec95233686451b0e
@@ -300,6 +300,28 @@ export const decodeGitFilePath = function (filePath: string): string {
   return filePath;
 };
 
+/**
+ * Get the root of the Git repository.
+ *
+ * @param sourceRoot The source root of the code being analyzed.
+ * @returns The root of the Git repository.
+ */
+export const getGitRoot = async function (
+  sourceRoot: string,
+): Promise<string | undefined> {
+  try {
+    const stdout = await runGitCommand(
+      sourceRoot,
+      ["rev-parse", "--show-toplevel"],
+      `Cannot find Git repository root from the source root ${sourceRoot}.`,
+    );
+    return stdout.trim();
+  } catch {
+    // Errors are already logged by runGitCommand()
+    return undefined;
+  }
+};
+
 function getRefFromEnv(): string {
   // To workaround a limitation of Actions dynamic workflows not setting
   // the GITHUB_REF in some cases, we accept also the ref within the

@@ -36,6 +36,7 @@ import { Feature, featureConfig, Features } from "./feature-flags";
 import {
   checkInstallPython311,
   cleanupDatabaseClusterDirectory,
+  getOverlayDatabaseMode,
   initCodeQL,
   initConfig,
   runInit,
@@ -396,7 +397,22 @@ async function run() {
   }
 
   try {
-    cleanupDatabaseClusterDirectory(config, logger);
+    const sourceRoot = path.resolve(
+      getRequiredEnvParam("GITHUB_WORKSPACE"),
+      getOptionalInput("source-root") || "",
+    );
+
+    const overlayDatabaseMode = await getOverlayDatabaseMode(
+      (await codeql.getVersion()).version,
+      config,
+      sourceRoot,
+      logger,
+    );
+    logger.info(`Using overlay database mode: ${overlayDatabaseMode}`);
+
+    if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
+      cleanupDatabaseClusterDirectory(config, logger);
+    }
 
     if (zstdAvailability) {
       await recordZstdAvailability(config, zstdAvailability);
@@ -676,19 +692,14 @@ async function run() {
       }
     }
 
-    const sourceRoot = path.resolve(
-      getRequiredEnvParam("GITHUB_WORKSPACE"),
-      getOptionalInput("source-root") || "",
-    );
-
     const tracerConfig = await runInit(
       codeql,
       config,
       sourceRoot,
       "Runner.Worker.exe",
       getOptionalInput("registries"),
       apiDetails,
-      OverlayDatabaseMode.None,
+      overlayDatabaseMode,
       logger,
     );
     if (tracerConfig !== undefined) {

@@ -3,15 +3,20 @@ import * as path from "path";
 
 import * as toolrunner from "@actions/exec/lib/toolrunner";
 import * as io from "@actions/io";
+import * as semver from "semver";
 
 import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
 import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
 import { CodeQL, setupCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
+import { getGitRoot } from "./git-utils";
 import { Language, isScannedLanguage } from "./languages";
 import { Logger } from "./logging";
-import { OverlayDatabaseMode } from "./overlay-database-utils";
+import {
+  CODEQL_OVERLAY_MINIMUM_VERSION,
+  OverlayDatabaseMode,
+} from "./overlay-database-utils";
 import { ToolsSource } from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
 import { ToolsDownloadStatusReport } from "./tools-download";
@@ -80,6 +85,47 @@ export async function initConfig(
   return config;
 }
 
+export async function getOverlayDatabaseMode(
+  codeqlVersion: string,
+  config: configUtils.Config,
+  sourceRoot: string,
+  logger: Logger,
+): Promise<OverlayDatabaseMode> {
+  const overlayDatabaseMode = process.env.CODEQL_OVERLAY_DATABASE_MODE;
+
+  if (
+    overlayDatabaseMode === OverlayDatabaseMode.Overlay ||
+    overlayDatabaseMode === OverlayDatabaseMode.OverlayBase
+  ) {
+    if (config.buildMode !== util.BuildMode.None) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `build-mode is set to "${config.buildMode}" instead of "none". ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    if (semver.lt(codeqlVersion, CODEQL_OVERLAY_MINIMUM_VERSION)) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `the CodeQL CLI is older than ${CODEQL_OVERLAY_MINIMUM_VERSION}. ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    if ((await getGitRoot(sourceRoot)) === undefined) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `the source root "${sourceRoot}" is not inside a git repository. ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    return overlayDatabaseMode as OverlayDatabaseMode;
+  }
+  return OverlayDatabaseMode.None;
+}
+
 export async function runInit(
   codeql: CodeQL,
   config: configUtils.Config,

@@ -3,3 +3,5 @@ export enum OverlayDatabaseMode {
   OverlayBase = "overlay-base",
   None = "none",
 }
+
+export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5";