diff --git a/extensions/ql-vscode/src/codeql-cli/cli-version.ts b/extensions/ql-vscode/src/codeql-cli/cli-version.ts index 5d3f7305688..ebd30cbdb62 100644 --- a/extensions/ql-vscode/src/codeql-cli/cli-version.ts +++ b/extensions/ql-vscode/src/codeql-cli/cli-version.ts @@ -10,9 +10,6 @@ interface VersionResult { } export interface CliFeatures { - featuresInVersionResult?: boolean; - mrvaPackCreate?: boolean; - generateSummarySymbolMap?: boolean; queryServerRunQueries?: boolean; } diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts index a34b6a351ea..f32c4a2555b 100644 --- a/extensions/ql-vscode/src/codeql-cli/cli.ts +++ b/extensions/ql-vscode/src/codeql-cli/cli.ts @@ -1191,15 +1191,12 @@ export class CodeQLCliServer implements Disposable { outputPath: string, endSummaryPath: string, ): Promise { - const supportsGenerateSummarySymbolMap = - await this.cliConstraints.supportsGenerateSummarySymbolMap(); const subcommandArgs = [ "--format=text", `--end-summary=${endSummaryPath}`, "--sourcemap", - ...(supportsGenerateSummarySymbolMap - ? ["--summary-symbol-map", "--minify-output"] - : []), + "--summary-symbol-map", + "--minify-output", inputPath, outputPath, ]; @@ -1910,11 +1907,7 @@ export class CliVersionConstraint { /**/ } - async supportsMrvaPackCreate(): Promise { - return (await this.cli.getFeatures()).mrvaPackCreate === true; - } - - async supportsGenerateSummarySymbolMap(): Promise { - return (await this.cli.getFeatures()).generateSummarySymbolMap === true; + async supportsQueryServerRunQueries(): Promise { + return (await this.cli.getFeatures()).queryServerRunQueries === true; } } diff --git a/extensions/ql-vscode/src/log-insights/summary-parser.ts b/extensions/ql-vscode/src/log-insights/summary-parser.ts index f00923e53ea..61a3a8af250 100644 --- a/extensions/ql-vscode/src/log-insights/summary-parser.ts +++ b/extensions/ql-vscode/src/log-insights/summary-parser.ts @@ -1,6 +1,3 @@ -import { createReadStream, writeFile } from "fs-extra"; -import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream"; - /** * Location information for a single pipeline invocation in the RA. */ @@ -32,103 +29,3 @@ interface PredicateSymbol { export interface SummarySymbols { predicates: Record; } - -// Tuple counts for Expr::Expr::getParent#dispred#f0820431#ff@76d6745o: -const NON_RECURSIVE_TUPLE_COUNT_REGEXP = - /^Evaluated relational algebra for predicate (?\S+) with tuple counts:$/; -// Tuple counts for Expr::Expr::getEnclosingStmt#f0820431#bf@923ddwj9 on iteration 0 running pipeline base: -const RECURSIVE_TUPLE_COUNT_REGEXP = - /^Evaluated relational algebra for predicate (?\S+) on iteration (?\d+) running pipeline (?\S+) with tuple counts:$/; -const RETURN_REGEXP = /^\s*return /; - -/** - * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline - * run. - * - * TODO: Once we're more certain about the symbol format, we should have the CLI generate this as it - * generates the human-readabe summary to avoid having to rely on regular expression matching of the - * human-readable text. - * - * @param summaryPath The path to the summary file. - * @param symbolsPath The path to the symbols file to generate. - */ -export async function generateSummarySymbolsFile( - summaryPath: string, - symbolsPath: string, -): Promise { - const symbols = await generateSummarySymbols(summaryPath); - await writeFile(symbolsPath, JSON.stringify(symbols)); -} - -/** - * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline - * run. - * - * @param fileLocation The path to the summary file. - * @returns Symbol information for the summary file. - */ -async function generateSummarySymbols( - summaryPath: string, -): Promise { - const stream = createReadStream(summaryPath, { - encoding: "utf-8", - }); - try { - const lines = splitStreamAtSeparators(stream, LINE_ENDINGS); - - const symbols: SummarySymbols = { - predicates: {}, - }; - - let lineNumber = 0; - let raStartLine = 0; - let iteration = 0; - let predicateName: string | undefined = undefined; - let startLine = 0; - for await (const line of lines) { - if (predicateName === undefined) { - // Looking for the start of the predicate. - const nonRecursiveMatch = line.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP); - if (nonRecursiveMatch) { - iteration = 0; - predicateName = nonRecursiveMatch.groups!.predicateName; - } else { - const recursiveMatch = line.match(RECURSIVE_TUPLE_COUNT_REGEXP); - if (recursiveMatch?.groups) { - predicateName = recursiveMatch.groups.predicateName; - iteration = parseInt(recursiveMatch.groups.iteration); - } - } - if (predicateName !== undefined) { - startLine = lineNumber; - raStartLine = lineNumber + 1; - } - } else { - const returnMatch = line.match(RETURN_REGEXP); - if (returnMatch) { - let symbol = symbols.predicates[predicateName]; - if (symbol === undefined) { - symbol = { - iterations: {}, - recursionSummaries: {}, - }; - symbols.predicates[predicateName] = symbol; - } - symbol.iterations[iteration] = { - startLine, - raStartLine, - raEndLine: lineNumber, - }; - - predicateName = undefined; - } - } - - lineNumber++; - } - - return symbols; - } finally { - stream.close(); - } -} diff --git a/extensions/ql-vscode/src/query-server/query-server-client.ts b/extensions/ql-vscode/src/query-server/query-server-client.ts index d52af580dea..8087686992f 100644 --- a/extensions/ql-vscode/src/query-server/query-server-client.ts +++ b/extensions/ql-vscode/src/query-server/query-server-client.ts @@ -100,7 +100,7 @@ export class QueryServerClient extends DisposableObject { * queries at once. */ async supportsRunQueriesMethod(): Promise { - return (await this.cliServer.getFeatures()).queryServerRunQueries === true; + return await this.cliServer.cliConstraints.supportsQueryServerRunQueries(); } /** Stops the query server by disposing of the current server process. */ diff --git a/extensions/ql-vscode/src/run-queries-shared.ts b/extensions/ql-vscode/src/run-queries-shared.ts index 990d3d8293d..4e463812cb0 100644 --- a/extensions/ql-vscode/src/run-queries-shared.ts +++ b/extensions/ql-vscode/src/run-queries-shared.ts @@ -27,7 +27,6 @@ import type { import type { BaseLogger } from "./common/logging"; import { showAndLogWarningMessage } from "./common/logging"; import { extLogger } from "./common/logging/vscode"; -import { generateSummarySymbolsFile } from "./log-insights/summary-parser"; import { getErrorMessage } from "./common/helpers-pure"; import { createHash } from "crypto"; import { QueryOutputDir } from "./local-queries/query-output-dir"; @@ -570,15 +569,6 @@ export async function generateEvalLogSummaries( if (humanReadableSummary !== undefined) { summarySymbols = outputDir.evalLogSummarySymbolsPath; - if ( - !(await cliServer.cliConstraints.supportsGenerateSummarySymbolMap()) - ) { - // We're using an old CLI that cannot generate the summary symbols file while generating the - // human-readable log summary. As a fallback, create it by parsing the human-readable - // summary. - progress(progressUpdate(3, 3, "Generating summary symbols file")); - await generateSummarySymbolsFile(humanReadableSummary, summarySymbols); - } } } diff --git a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts index e7e67a0df6c..5b29eed8fd0 100644 --- a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts +++ b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts @@ -1,7 +1,7 @@ import type { CancellationToken } from "vscode"; -import { Uri, window } from "vscode"; -import { join, sep, basename, relative } from "path"; -import { dump, load } from "js-yaml"; +import { window } from "vscode"; +import { join, basename, relative } from "path"; +import { dump } from "js-yaml"; import { copy, writeFile, readFile, mkdirp, remove } from "fs-extra"; import { nanoid } from "nanoid"; import { tmpDir } from "../tmp-dir"; @@ -27,13 +27,7 @@ import { } from "./repository-selection"; import type { Repository } from "./shared/repository"; import type { DbManager } from "../databases/db-manager"; -import { - getQlPackFilePath, - FALLBACK_QLPACK_FILENAME, - QLPACK_FILENAMES, - QLPACK_LOCK_FILENAMES, -} from "../common/ql"; -import type { QlPackFile } from "../packaging/qlpack-file"; +import { FALLBACK_QLPACK_FILENAME } from "../common/ql"; import { expandShortPaths } from "../common/short-paths"; import type { QlPackDetails } from "./ql-pack-details"; import type { ModelPackDetails } from "../common/model-pack-details"; @@ -68,8 +62,6 @@ async function generateQueryPack( ); const mustSynthesizePack = qlPackDetails.qlPackFilePath === undefined; - const cliSupportsMrvaPackCreate = - await cliServer.cliConstraints.supportsMrvaPackCreate(); let targetPackPath: string; let needsInstall: boolean; @@ -87,15 +79,6 @@ async function generateQueryPack( // Install packs, since we just synthesized a dependency on the language's standard library. needsInstall = true; - } else if (!cliSupportsMrvaPackCreate) { - // We need to copy the query pack to a temporary directory and then fix it up to work with MRVA. - targetPackPath = tmpDir.queryPackDir; - await copyExistingQueryPack(cliServer, qlPackDetails, targetPackPath); - - // We should already have all the dependencies available, but these older versions of the CLI - // have a bug where they will not search `--additional-packs` during validation in `codeql pack bundle`. - // Installing the packs will ensure that any extension packs get put in the right place. - needsInstall = true; } else { // The CLI supports creating a MRVA query pack directly from the source pack. targetPackPath = qlPackDetails.qlPackRootPath; @@ -113,27 +96,18 @@ async function generateQueryPack( await cliServer.clearCache(); } - let precompilationOpts: string[]; - if (cliSupportsMrvaPackCreate) { - const queryOpts = qlPackDetails.queryFiles.flatMap((q) => [ - "--query", - join(targetPackPath, relative(qlPackDetails.qlPackRootPath, q)), - ]); - - precompilationOpts = [ - "--mrva", - ...queryOpts, - // We need to specify the extension packs as dependencies so that they are included in the MRVA pack. - // The version range doesn't matter, since they'll always be found by source lookup. - ...extensionPacks.map((p) => `--extension-pack=${p.name}@*`), - ]; - } else { - precompilationOpts = ["--qlx"]; - - if (extensionPacks.length > 0) { - await addExtensionPacksAsDependencies(targetPackPath, extensionPacks); - } - } + const queryOpts = qlPackDetails.queryFiles.flatMap((q) => [ + "--query", + join(targetPackPath, relative(qlPackDetails.qlPackRootPath, q)), + ]); + + const precompilationOpts = [ + "--mrva", + ...queryOpts, + // We need to specify the extension packs as dependencies so that they are included in the MRVA pack. + // The version range doesn't matter, since they'll always be found by source lookup. + ...extensionPacks.map((p) => `--extension-pack=${p.name}@*`), + ]; const bundlePath = tmpDir.bundleFile; void extLogger.log( @@ -181,59 +155,6 @@ async function createNewQueryPack( ); } -async function copyExistingQueryPack( - cliServer: CodeQLCliServer, - qlPackDetails: QlPackDetails, - targetPackPath: string, -) { - const toCopy = await cliServer.packPacklist( - qlPackDetails.qlPackRootPath, - false, - ); - - // Also include query files that contain extensible predicates. These query files are not - // needed for the query to run, but they are needed for the query pack to pass deep validation - // of data extensions. - const metadata = await cliServer.generateExtensiblePredicateMetadata( - qlPackDetails.qlPackRootPath, - ); - metadata.extensible_predicates.forEach((predicate) => { - if (predicate.path.endsWith(".ql")) { - toCopy.push(join(qlPackDetails.qlPackRootPath, predicate.path)); - } - }); - - [ - // also copy the lock file (either new name or old name) and the query file itself. These are not included in the packlist. - ...QLPACK_LOCK_FILENAMES.map((f) => join(qlPackDetails.qlPackRootPath, f)), - ...qlPackDetails.queryFiles, - ].forEach((absolutePath) => { - if (absolutePath) { - toCopy.push(absolutePath); - } - }); - - let copiedCount = 0; - await copy(qlPackDetails.qlPackRootPath, targetPackPath, { - filter: (file: string) => - // copy file if it is in the packlist, or it is a parent directory of a file in the packlist - !!toCopy.find((f) => { - // Normalized paths ensure that Windows drive letters are capitalized consistently. - const normalizedPath = Uri.file(f).fsPath; - const matches = - normalizedPath === file || normalizedPath.startsWith(file + sep); - if (matches) { - copiedCount++; - } - return matches; - }), - }); - - void extLogger.log(`Copied ${copiedCount} files to ${targetPackPath}`); - - await fixPackFile(targetPackPath, qlPackDetails); -} - interface RemoteQueryTempDir { remoteQueryDir: string; queryPackDir: string; @@ -351,42 +272,6 @@ export async function prepareRemoteQueryRun( }; } -/** - * Fixes the qlpack.yml or codeql-pack.yml file to be correct in the context of the MRVA request. - * - * Performs the following fixes: - * - * - Updates the default suite of the query pack. This is used to ensure - * only the specified query is run. - * - Ensures the query pack name is set to the name expected by the server. - * - Removes any `${workspace}` version references from the qlpack.yml or codeql-pack.yml file. Converts them - * to `*` versions. - * - * @param targetPackPath The path to the directory containing the target pack - * @param qlPackDetails The details of the original QL pack - */ -async function fixPackFile( - targetPackPath: string, - qlPackDetails: QlPackDetails, -): Promise { - const packPath = await getQlPackFilePath(targetPackPath); - - // This should not happen since we create the pack ourselves. - if (!packPath) { - throw new Error( - `Could not find ${QLPACK_FILENAMES.join( - " or ", - )} file in '${targetPackPath}'`, - ); - } - const qlpack = load(await readFile(packPath, "utf8")) as QlPackFile; - - updateDefaultSuite(qlpack, qlPackDetails); - removeWorkspaceRefs(qlpack); - - await writeFile(packPath, dump(qlpack)); -} - async function getExtensionPacksToInject( cliServer: CodeQLCliServer, workspaceFolders: string[], @@ -415,41 +300,6 @@ async function getExtensionPacksToInject( return result; } -async function addExtensionPacksAsDependencies( - queryPackDir: string, - extensionPacks: ModelPackDetails[], -): Promise { - const qlpackFile = await getQlPackFilePath(queryPackDir); - if (!qlpackFile) { - throw new Error( - `Could not find ${QLPACK_FILENAMES.join( - " or ", - )} file in '${queryPackDir}'`, - ); - } - - const syntheticQueryPack = load( - await readFile(qlpackFile, "utf8"), - ) as QlPackFile; - - const dependencies = syntheticQueryPack.dependencies ?? {}; - extensionPacks.forEach(({ name }) => { - // Add this extension pack as a dependency. It doesn't matter which - // version we specify, since we are guaranteed that the extension pack - // is resolved from source at the given path. - dependencies[name] = "*"; - }); - - syntheticQueryPack.dependencies = dependencies; - - await writeFile(qlpackFile, dump(syntheticQueryPack)); -} - -function updateDefaultSuite(qlpack: QlPackFile, qlPackDetails: QlPackDetails) { - delete qlpack.defaultSuiteFile; - qlpack.defaultSuite = generateDefaultSuite(qlPackDetails); -} - function generateDefaultSuite(qlPackDetails: QlPackDetails) { const queries = qlPackDetails.queryFiles.map((query) => { const relativePath = relative(qlPackDetails.qlPackRootPath, query); @@ -550,15 +400,3 @@ async function getControllerRepoFromApi( } } } - -function removeWorkspaceRefs(qlpack: QlPackFile) { - if (!qlpack.dependencies) { - return; - } - - for (const [key, value] of Object.entries(qlpack.dependencies)) { - if (value === "${workspace}") { - qlpack.dependencies[key] = "*"; - } - } -} pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy