Skip to content

Commit c702abb

Browse files
cklincklin
committed
Support overlay database creation
1 parent c261e0e commit c702abb

File tree

8 files changed

+286
-7
lines changed

8 files changed

+286
-7
lines changed

‎src/analyze.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ test("status report fields", async (t) => {
101101
memoryFlag,
102102
addSnippetsFlag,
103103
threadsFlag,
104+
"brutal",
104105
undefined,
105106
undefined,
106107
config,

‎src/codeql.test.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import { DocUrl } from "./doc-url";
2020
import { FeatureEnablement } from "./feature-flags";
2121
import { Language } from "./languages";
2222
import { getRunnerLogger } from "./logging";
23+
import { OverlayDatabaseMode } from "./overlay-database-utils";
2324
import { ToolsSource } from "./setup-codeql";
2425
import {
2526
setupTests,
@@ -510,6 +511,7 @@ const injectedConfigMacro = test.macro({
510511
"",
511512
undefined,
512513
undefined,
514+
OverlayDatabaseMode.None,
513515
getRunnerLogger(true),
514516
);
515517

@@ -723,6 +725,7 @@ test("passes a code scanning config AND qlconfig to the CLI", async (t: Executio
723725
"",
724726
undefined,
725727
"/path/to/qlconfig.yml",
728+
OverlayDatabaseMode.None,
726729
getRunnerLogger(true),
727730
);
728731

@@ -752,6 +755,7 @@ test("does not pass a qlconfig to the CLI when it is undefined", async (t: Execu
752755
"",
753756
undefined,
754757
undefined, // undefined qlconfigFile
758+
OverlayDatabaseMode.None,
755759
getRunnerLogger(true),
756760
);
757761

@@ -1005,6 +1009,7 @@ test("Avoids duplicating --overwrite flag if specified in CODEQL_ACTION_EXTRA_OP
10051009
"sourceRoot",
10061010
undefined,
10071011
undefined,
1012+
OverlayDatabaseMode.None,
10081013
getRunnerLogger(false),
10091014
);
10101015

‎src/codeql.ts

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ import {
2424
import { isAnalyzingDefaultBranch } from "./git-utils";
2525
import { Language } from "./languages";
2626
import { Logger } from "./logging";
27+
import {
28+
OverlayDatabaseMode,
29+
writeBaseDatabaseOidsFile,
30+
writeOverlayChangedFilesFile,
31+
} from "./overlay-database-utils";
2732
import * as setupCodeql from "./setup-codeql";
2833
import { ZstdAvailability } from "./tar";
2934
import { ToolsDownloadStatusReport } from "./tools-download";
@@ -82,6 +87,7 @@ export interface CodeQL {
8287
sourceRoot: string,
8388
processName: string | undefined,
8489
qlconfigFile: string | undefined,
90+
overlayDatabaseMode: OverlayDatabaseMode,
8591
logger: Logger,
8692
): Promise<void>;
8793
/**
@@ -552,6 +558,7 @@ export async function getCodeQLForCmd(
552558
sourceRoot: string,
553559
processName: string | undefined,
554560
qlconfigFile: string | undefined,
561+
overlayDatabaseMode: OverlayDatabaseMode,
555562
logger: Logger,
556563
) {
557564
const extraArgs = config.languages.map(
@@ -606,12 +613,21 @@ export async function getCodeQLForCmd(
606613
? "--force-overwrite"
607614
: "--overwrite";
608615

616+
if (overlayDatabaseMode === OverlayDatabaseMode.Overlay) {
617+
await writeOverlayChangedFilesFile(config, sourceRoot, logger);
618+
extraArgs.push("--overlay");
619+
} else if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
620+
extraArgs.push("--overlay-base");
621+
}
622+
609623
await runCli(
610624
cmd,
611625
[
612626
"database",
613627
"init",
614-
overwriteFlag,
628+
...(overlayDatabaseMode === OverlayDatabaseMode.Overlay
629+
? []
630+
: [overwriteFlag]),
615631
"--db-cluster",
616632
config.dbLocation,
617633
`--source-root=${sourceRoot}`,
@@ -625,6 +641,10 @@ export async function getCodeQLForCmd(
625641
],
626642
{ stdin: externalRepositoryToken },
627643
);
644+
645+
if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
646+
await writeBaseDatabaseOidsFile(config, sourceRoot);
647+
}
628648
},
629649
async runAutobuild(config: Config, language: Language) {
630650
applyAutobuildAzurePipelinesTimeoutFix();

‎src/git-utils.ts

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,61 @@ export const decodeGitFilePath = function (filePath: string): string {
300300
return filePath;
301301
};
302302

303+
/**
304+
* Get the oids of all files in HEAD.
305+
*
306+
* @param checkoutPath A path into the Git repository.
307+
* @returns a map from file paths to the corresponding oid.
308+
* @throws {Error} if "git ls-tree" produces unexpected output.
309+
*/
310+
export const getAllFileOids = async function (
311+
checkoutPath: string,
312+
): Promise<{ [key: string]: string }> {
313+
const stdout = await runGitCommand(
314+
checkoutPath,
315+
["ls-tree", "--format=%(objectname)_%(path)", "-r", "HEAD"],
316+
"Cannot list file OIDs in HEAD.",
317+
);
318+
319+
const fileOidMap: { [key: string]: string } = {};
320+
const regex = /^([0-9a-f]{40})_(.+)$/;
321+
for (const line of stdout.split("\n")) {
322+
if (line) {
323+
const match = line.match(regex);
324+
if (match) {
325+
const oid = match[1];
326+
const path = decodeGitFilePath(match[2]);
327+
fileOidMap[path] = oid;
328+
} else {
329+
throw new Error(`Unexpected "git ls-tree" output: ${line}`);
330+
}
331+
}
332+
}
333+
return fileOidMap;
334+
};
335+
336+
/**
337+
* Get the root of the Git repository.
338+
*
339+
* @param sourceRoot The source root of the code being analyzed.
340+
* @returns The root of the Git repository.
341+
*/
342+
export const getGitRoot = async function (
343+
sourceRoot: string,
344+
): Promise<string | undefined> {
345+
try {
346+
const stdout = await runGitCommand(
347+
sourceRoot,
348+
["rev-parse", "--show-toplevel"],
349+
`Cannot find Git repository root from the source root ${sourceRoot}.`,
350+
);
351+
return stdout.trim();
352+
} catch {
353+
// Errors are already logged by runGitCommand()
354+
return undefined;
355+
}
356+
};
357+
303358
function getRefFromEnv(): string {
304359
// To workaround a limitation of Actions dynamic workflows not setting
305360
// the GITHUB_REF in some cases, we accept also the ref within the

‎src/init-action.ts

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,14 @@ import { Feature, featureConfig, Features } from "./feature-flags";
3636
import {
3737
checkInstallPython311,
3838
cleanupDatabaseClusterDirectory,
39+
getOverlayDatabaseMode,
3940
initCodeQL,
4041
initConfig,
4142
runInit,
4243
} from "./init";
4344
import { Language } from "./languages";
4445
import { getActionsLogger, Logger } from "./logging";
46+
import { OverlayDatabaseMode } from "./overlay-database-utils";
4547
import { parseRepositoryNwo } from "./repository";
4648
import { ToolsSource } from "./setup-codeql";
4749
import {
@@ -395,7 +397,22 @@ async function run() {
395397
}
396398

397399
try {
398-
cleanupDatabaseClusterDirectory(config, logger);
400+
const sourceRoot = path.resolve(
401+
getRequiredEnvParam("GITHUB_WORKSPACE"),
402+
getOptionalInput("source-root") || "",
403+
);
404+
405+
const overlayDatabaseMode = await getOverlayDatabaseMode(
406+
(await codeql.getVersion()).version,
407+
config,
408+
sourceRoot,
409+
logger,
410+
);
411+
logger.info(`Using overlay database mode: ${overlayDatabaseMode}`);
412+
413+
if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
414+
cleanupDatabaseClusterDirectory(config, logger);
415+
}
399416

400417
if (zstdAvailability) {
401418
await recordZstdAvailability(config, zstdAvailability);
@@ -675,18 +692,14 @@ async function run() {
675692
}
676693
}
677694

678-
const sourceRoot = path.resolve(
679-
getRequiredEnvParam("GITHUB_WORKSPACE"),
680-
getOptionalInput("source-root") || "",
681-
);
682-
683695
const tracerConfig = await runInit(
684696
codeql,
685697
config,
686698
sourceRoot,
687699
"Runner.Worker.exe",
688700
getOptionalInput("registries"),
689701
apiDetails,
702+
overlayDatabaseMode,
690703
logger,
691704
);
692705
if (tracerConfig !== undefined) {

‎src/init.ts

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,20 @@ import * as path from "path";
33

44
import * as toolrunner from "@actions/exec/lib/toolrunner";
55
import * as io from "@actions/io";
6+
import * as semver from "semver";
67

78
import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
89
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
910
import { CodeQL, setupCodeQL } from "./codeql";
1011
import * as configUtils from "./config-utils";
1112
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
13+
import { getGitRoot } from "./git-utils";
1214
import { Language, isScannedLanguage } from "./languages";
1315
import { Logger } from "./logging";
16+
import {
17+
CODEQL_OVERLAY_MINIMUM_VERSION,
18+
OverlayDatabaseMode,
19+
} from "./overlay-database-utils";
1420
import { ToolsSource } from "./setup-codeql";
1521
import { ZstdAvailability } from "./tar";
1622
import { ToolsDownloadStatusReport } from "./tools-download";
@@ -79,13 +85,55 @@ export async function initConfig(
7985
return config;
8086
}
8187

88+
export async function getOverlayDatabaseMode(
89+
codeqlVersion: string,
90+
config: configUtils.Config,
91+
sourceRoot: string,
92+
logger: Logger,
93+
): Promise<OverlayDatabaseMode> {
94+
const overlayDatabaseMode = process.env.CODEQL_OVERLAY_DATABASE_MODE;
95+
96+
if (
97+
overlayDatabaseMode === OverlayDatabaseMode.Overlay ||
98+
overlayDatabaseMode === OverlayDatabaseMode.OverlayBase
99+
) {
100+
if (config.buildMode !== util.BuildMode.None) {
101+
logger.warning(
102+
`Cannot build an ${overlayDatabaseMode} database because ` +
103+
`build-mode is set to "${config.buildMode}" instead of "none". ` +
104+
"Falling back to creating a normal full database instead.",
105+
);
106+
return OverlayDatabaseMode.None;
107+
}
108+
if (semver.lt(codeqlVersion, CODEQL_OVERLAY_MINIMUM_VERSION)) {
109+
logger.warning(
110+
`Cannot build an ${overlayDatabaseMode} database because ` +
111+
`the CodeQL CLI is older than ${CODEQL_OVERLAY_MINIMUM_VERSION}. ` +
112+
"Falling back to creating a normal full database instead.",
113+
);
114+
return OverlayDatabaseMode.None;
115+
}
116+
if ((await getGitRoot(sourceRoot)) === undefined) {
117+
logger.warning(
118+
`Cannot build an ${overlayDatabaseMode} database because ` +
119+
`the source root "${sourceRoot}" is not inside a git repository. ` +
120+
"Falling back to creating a normal full database instead.",
121+
);
122+
return OverlayDatabaseMode.None;
123+
}
124+
return overlayDatabaseMode as OverlayDatabaseMode;
125+
}
126+
return OverlayDatabaseMode.None;
127+
}
128+
82129
export async function runInit(
83130
codeql: CodeQL,
84131
config: configUtils.Config,
85132
sourceRoot: string,
86133
processName: string | undefined,
87134
registriesInput: string | undefined,
88135
apiDetails: GitHubApiCombinedDetails,
136+
overlayDatabaseMode: OverlayDatabaseMode,
89137
logger: Logger,
90138
): Promise<TracerConfig | undefined> {
91139
fs.mkdirSync(config.dbLocation, { recursive: true });
@@ -109,6 +157,7 @@ export async function runInit(
109157
sourceRoot,
110158
processName,
111159
qlconfigFile,
160+
overlayDatabaseMode,
112161
logger,
113162
),
114163
);

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy