Overlay: report overlay-base database stats

github · cklin · Jul 21, 2025 · Jul 18, 2025 · Jul 18, 2025 · Jul 17, 2025
commit e37b293334be08308a49bc95fc999bbbb27bb1f8
@@ -44,6 +44,7 @@ import { Language } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
 import {
   downloadOverlayBaseDatabaseFromCache,
+  OverlayBaseDatabaseDownloadStats,
   OverlayDatabaseMode,
 } from "./overlay-database-utils";
 import { getRepositoryNwo } from "./repository";
@@ -107,6 +108,10 @@ interface InitWithConfigStatusReport extends InitStatusReport {
   trap_cache_download_size_bytes: number;
   /** Time taken to download TRAP caches, in milliseconds. */
   trap_cache_download_duration_ms: number;
+  /** Size of the overlay-base database that we downloaded, in bytes. */
+  overlay_base_database_download_size_bytes?: number;
+  /** Time taken to download the overlay-base database, in milliseconds. */
+  overlay_base_database_download_duration_ms?: number;
   /** Stringified JSON array of registry configuration objects, from the 'registries' config field
   or workflow input. **/
   registries: string;
@@ -134,6 +139,7 @@ async function sendCompletedStatusReport(
   toolsFeatureFlagsValid: boolean | undefined,
   toolsSource: ToolsSource,
   toolsVersion: string,
+  overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined,
   logger: Logger,
   error?: Error,
 ) {
@@ -237,6 +243,10 @@ async function sendCompletedStatusReport(
         await getTotalCacheSize(Object.values(config.trapCaches), logger),
       ),
       trap_cache_download_duration_ms: Math.round(config.trapCacheDownloadTime),
+      overlay_base_database_download_size_bytes:
+        overlayBaseDatabaseStats?.databaseSizeBytes,
+      overlay_base_database_download_duration_ms:
+        overlayBaseDatabaseStats?.databaseDownloadDurationMs,
       query_filters: JSON.stringify(
         config.originalUserInput["query-filters"] ?? [],
       ),
@@ -400,6 +410,7 @@ async function run() {
     return;
   }
 
+  let overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined;
   try {
     if (
       config.augmentationProperties.overlayDatabaseMode ===
@@ -417,9 +428,12 @@ async function run() {
       // necessary preparations. So, in that mode, we would assume that
       // everything is in order and let the analysis fail if that turns out not
       // to be the case.
-      const overlayDatabaseDownloaded =
-        await downloadOverlayBaseDatabaseFromCache(codeql, config, logger);
-      if (!overlayDatabaseDownloaded) {
+      overlayBaseDatabaseStats = await downloadOverlayBaseDatabaseFromCache(
+        codeql,
+        config,
+        logger,
+      );
+      if (!overlayBaseDatabaseStats) {
         config.augmentationProperties.overlayDatabaseMode =
           OverlayDatabaseMode.None;
         logger.info(
@@ -729,6 +743,7 @@ async function run() {
       toolsFeatureFlagsValid,
       toolsSource,
       toolsVersion,
+      overlayBaseDatabaseStats,
       logger,
       error,
     );
@@ -744,6 +759,7 @@ async function run() {
     toolsFeatureFlagsValid,
     toolsSource,
     toolsVersion,
+    overlayBaseDatabaseStats,
     logger,
   );
 }

@@ -8,7 +8,7 @@ import { type CodeQL } from "./codeql";
 import { type Config } from "./config-utils";
 import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
 import { Logger } from "./logging";
-import { isInTestMode, withTimeout } from "./util";
+import { isInTestMode, tryGetFolderBytes, withTimeout } from "./util";
 
 export enum OverlayDatabaseMode {
   Overlay = "overlay",
@@ -235,41 +235,47 @@ export async function uploadOverlayBaseDatabaseToCache(
   return true;
 }
 
+export interface OverlayBaseDatabaseDownloadStats {
+  databaseSizeBytes: number;
+  databaseDownloadDurationMs: number;
+}
+
 /**
  * Downloads the overlay-base database from the GitHub Actions cache. If conditions
  * for downloading are not met, the function does nothing and returns false.
  *
  * @param codeql The CodeQL instance
  * @param config The configuration object
  * @param logger The logger instance
- * @returns A promise that resolves to true if the download was performed and
- * successfully completed, or false otherwise
+ * @returns A promise that resolves to download statistics if an overlay-base
+ * database was successfully downloaded, or undefined if the download was
+ * either not performed or failed.
  */
 export async function downloadOverlayBaseDatabaseFromCache(
   codeql: CodeQL,
   config: Config,
   logger: Logger,
-): Promise<boolean> {
+): Promise<OverlayBaseDatabaseDownloadStats | undefined> {
   const overlayDatabaseMode = config.augmentationProperties.overlayDatabaseMode;
   if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
     logger.debug(
       `Overlay database mode is ${overlayDatabaseMode}. ` +
         "Skip downloading overlay-base database from cache.",
     );
-    return false;
+    return undefined;
   }
   if (!config.augmentationProperties.useOverlayDatabaseCaching) {
     logger.debug(
       "Overlay database caching is disabled. " +
         "Skip downloading overlay-base database from cache.",
     );
-    return false;
+    return undefined;
   }
   if (isInTestMode()) {
     logger.debug(
       "In test mode. Skip downloading overlay-base database from cache.",
     );
-    return false;
+    return undefined;
   }
 
   const dbLocation = config.dbLocation;
@@ -280,18 +286,23 @@ export async function downloadOverlayBaseDatabaseFromCache(
     `Looking in Actions cache for overlay-base database with restore key ${restoreKey}`,
   );
 
+  let databaseDownloadDurationMs = 0;
   try {
+    const databaseDownloadStart = performance.now();
     const foundKey = await withTimeout(
       MAX_CACHE_OPERATION_MS,
       actionsCache.restoreCache([dbLocation], restoreKey),
       () => {
         logger.info("Timed out downloading overlay-base database from cache");
       },
     );
+    databaseDownloadDurationMs = Math.round(
+      performance.now() - databaseDownloadStart,
+    );
 
     if (foundKey === undefined) {
       logger.info("No overlay-base database found in Actions cache");
-      return false;
+      return undefined;
     }
 
     logger.info(
@@ -302,7 +313,7 @@ export async function downloadOverlayBaseDatabaseFromCache(
       "Failed to download overlay-base database from cache: " +
         `${error instanceof Error ? error.message : String(error)}`,
     );
-    return false;
+    return undefined;
   }
 
   const databaseIsValid = checkOverlayBaseDatabase(
@@ -312,11 +323,26 @@ export async function downloadOverlayBaseDatabaseFromCache(
   );
   if (!databaseIsValid) {
     logger.warning("Downloaded overlay-base database failed validation");
-    return false;
+    return undefined;
+  }
+
+  const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
+  if (databaseSizeBytes === undefined) {
+    logger.info(
+      "Filesystem error while accessing downloaded overlay-base database",
+    );
+    // The problem that warrants reporting download failure is not that we are
+    // unable to determine the size of the database. Rather, it is that we
+    // encountered a filesystem error while accessing the database, which
+    // indicates that an overlay analysis will likely fail.
+    return undefined;
   }
 
   logger.info(`Successfully downloaded overlay-base database to ${dbLocation}`);
-  return true;
+  return {
+    databaseSizeBytes: Math.round(databaseSizeBytes),
+    databaseDownloadDurationMs,
+  };
 }
 
 async function generateCacheKey(