Skip to content

Commit f6ed443

Browse files
gatsbybotpieh
andauthored
fix(gatsby): try to automatically recover when parcel segfaults (#38773) (#38799)
* fix(gatsby): try to automatically recover when parcel segfauls * test: make gatsby-worker test adjustment global * fix: handle actual compilation errors * test: bump timeout for windows * init bundles array so TS is happy (cherry picked from commit 0a80cd6) Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>
1 parent 68b0821 commit f6ed443

File tree

4 files changed

+173
-25
lines changed

4 files changed

+173
-25
lines changed

.jestSetup.js

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,30 @@ if (
55
typeof globalThis.TextEncoder === "undefined" ||
66
typeof globalThis.TextDecoder === "undefined"
77
) {
8-
const utils = require("util");
9-
globalThis.TextEncoder = utils.TextEncoder;
10-
globalThis.TextDecoder = utils.TextDecoder;
8+
const utils = require("util")
9+
globalThis.TextEncoder = utils.TextEncoder
10+
globalThis.TextDecoder = utils.TextDecoder
1111
}
12+
13+
jest.mock(`gatsby-worker`, () => {
14+
const gatsbyWorker = jest.requireActual(`gatsby-worker`)
15+
16+
const { WorkerPool: OriginalWorkerPool } = gatsbyWorker
17+
18+
class WorkerPoolThatCanUseTS extends OriginalWorkerPool {
19+
constructor(workerPath, options) {
20+
options.env = {
21+
...(options.env ?? {}),
22+
NODE_OPTIONS: `--require ${require.resolve(
23+
`./packages/gatsby/src/utils/worker/__tests__/test-helpers/ts-register.js`
24+
)}`,
25+
}
26+
super(workerPath, options)
27+
}
28+
}
29+
30+
return {
31+
...gatsbyWorker,
32+
WorkerPool: WorkerPoolThatCanUseTS,
33+
}
34+
})

packages/gatsby/src/utils/parcel/__tests__/compile-gatsby-files.ts

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@ const dir = {
1818
misnamedJS: `${__dirname}/fixtures/misnamed-js`,
1919
misnamedTS: `${__dirname}/fixtures/misnamed-ts`,
2020
gatsbyNodeAsDirectory: `${__dirname}/fixtures/gatsby-node-as-directory`,
21+
errorInCode: `${__dirname}/fixtures/error-in-code-ts`,
2122
}
2223

23-
jest.setTimeout(15000)
24+
jest.setTimeout(60_000)
2425

2526
jest.mock(`@parcel/core`, () => {
2627
const parcelCore = jest.requireActual(`@parcel/core`)
@@ -175,6 +176,37 @@ describe(`gatsby file compilation`, () => {
175176
})
176177
})
177178
})
179+
180+
it(`handles errors in TS code`, async () => {
181+
process.chdir(dir.errorInCode)
182+
await remove(`${dir.errorInCode}/.cache`)
183+
await compileGatsbyFiles(dir.errorInCode)
184+
185+
expect(reporterPanicMock).toMatchInlineSnapshot(`
186+
[MockFunction] {
187+
"calls": Array [
188+
Array [
189+
Object {
190+
"context": Object {
191+
"filePath": "<PROJECT_ROOT>/gatsby-node.ts",
192+
"generalMessage": "Expected ';', '}' or <eof>",
193+
"hints": null,
194+
"origin": "@parcel/transformer-js",
195+
"specificMessage": "This is the expression part of an expression statement",
196+
},
197+
"id": "11901",
198+
},
199+
],
200+
],
201+
"results": Array [
202+
Object {
203+
"type": "return",
204+
"value": undefined,
205+
},
206+
],
207+
}
208+
`)
209+
})
178210
})
179211

180212
describe(`gatsby-node directory is allowed`, () => {
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { GatsbyNode } from "gatsby"
2+
import { working } from "../utils/say-what-ts"
3+
import { createPages } from "../utils/create-pages-ts"
4+
5+
this is wrong syntax that should't compile
6+
7+
export const onPreInit: GatsbyNode["onPreInit"] = ({ reporter }) => {
8+
reporter.info(working)
9+
}
10+
11+
type Character = {
12+
id: string
13+
name: string
14+
}
15+
16+
export const sourceNodes: GatsbyNode["sourceNodes"] = async ({ actions, createNodeId, createContentDigest }) => {
17+
const { createNode } = actions
18+
19+
let characters: Array<Character> = [
20+
{
21+
id: `0`,
22+
name: `A`
23+
},
24+
{
25+
id: `1`,
26+
name: `B`
27+
}
28+
]
29+
30+
characters.forEach((character: Character) => {
31+
const node = {
32+
...character,
33+
id: createNodeId(`characters-${character.id}`),
34+
parent: null,
35+
children: [],
36+
internal: {
37+
type: 'Character',
38+
content: JSON.stringify(character),
39+
contentDigest: createContentDigest(character),
40+
},
41+
}
42+
43+
createNode(node)
44+
})
45+
}
46+
47+
export { createPages }

packages/gatsby/src/utils/parcel/compile-gatsby-files.ts

Lines changed: 67 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { LMDBCache, Cache } from "@parcel/cache"
33
import path from "path"
44
import type { Diagnostic } from "@parcel/diagnostic"
55
import reporter from "gatsby-cli/lib/reporter"
6+
import { WorkerPool } from "gatsby-worker"
67
import { ensureDir, emptyDir, existsSync, remove, readdir } from "fs-extra"
78
import telemetry from "gatsby-telemetry"
89
import { isNearMatch } from "../is-near-match"
@@ -52,6 +53,28 @@ export function constructParcel(siteRoot: string, cache?: Cache): Parcel {
5253
})
5354
}
5455

56+
interface IProcessBundle {
57+
filePath: string
58+
mainEntryPath?: string
59+
}
60+
61+
type RunParcelReturn = Array<IProcessBundle>
62+
63+
export async function runParcel(siteRoot: string): Promise<RunParcelReturn> {
64+
const cache = new LMDBCache(getCacheDir(siteRoot)) as unknown as Cache
65+
const parcel = constructParcel(siteRoot, cache)
66+
const { bundleGraph } = await parcel.run()
67+
const bundles = bundleGraph.getBundles()
68+
// bundles is not serializable, so we need to extract the data we need
69+
// so it crosses IPC boundaries
70+
return bundles.map(bundle => {
71+
return {
72+
filePath: bundle.filePath,
73+
mainEntryPath: bundle.getMainEntry()?.filePath,
74+
}
75+
})
76+
}
77+
5578
/**
5679
* Compile known gatsby-* files (e.g. `gatsby-config`, `gatsby-node`)
5780
* and output in `<SITE_ROOT>/.cache/compiled`.
@@ -107,33 +130,59 @@ export async function compileGatsbyFiles(
107130
})
108131
}
109132

133+
const worker = new WorkerPool<typeof import("./compile-gatsby-files")>(
134+
require.resolve(`./compile-gatsby-files`),
135+
{
136+
numWorkers: 1,
137+
}
138+
)
139+
110140
const distDir = `${siteRoot}/${COMPILED_CACHE_DIR}`
111141
await ensureDir(distDir)
112142
await emptyDir(distDir)
113143

114144
await exponentialBackoff(retry)
115145

116-
// for whatever reason TS thinks LMDBCache is some browser Cache and not actually Parcel's Cache
117-
// so we force type it to Parcel's Cache
118-
const cache = new LMDBCache(getCacheDir(siteRoot)) as unknown as Cache
119-
const parcel = constructParcel(siteRoot, cache)
120-
const { bundleGraph } = await parcel.run()
121-
let cacheClosePromise = Promise.resolve()
146+
let bundles: RunParcelReturn = []
122147
try {
123-
// @ts-ignore store is public field on LMDBCache class, but public interface for Cache
124-
// doesn't have it. There doesn't seem to be proper public API for this, so we have to
125-
// resort to reaching into internals. Just in case this is wrapped in try/catch if
126-
// parcel changes internals in future (closing cache is only needed when retrying
127-
// so the if the change happens we shouldn't fail on happy builds)
128-
cacheClosePromise = cache.store.close()
129-
} catch (e) {
130-
reporter.verbose(`Failed to close parcel cache\n${e.toString()}`)
148+
// sometimes parcel segfaults which is not something we can recover from, so we run parcel
149+
// in child process and IF it fails we try to delete parcel's cache (this seems to "fix" the problem
150+
// causing segfaults?) and retry few times
151+
// not ideal, but having gatsby segfaulting is really frustrating and common remedy is to clean
152+
// entire .cache for users, which is not ideal either especially when we can just delete parcel's cache
153+
// and to recover automatically
154+
bundles = await worker.single.runParcel(siteRoot)
155+
} catch (error) {
156+
if (error.diagnostics) {
157+
handleErrors(error.diagnostics)
158+
return
159+
} else if (retry >= RETRY_COUNT) {
160+
reporter.panic({
161+
id: `11904`,
162+
error,
163+
context: {
164+
siteRoot,
165+
retries: RETRY_COUNT,
166+
sourceMessage: error.message,
167+
},
168+
})
169+
} else {
170+
await exponentialBackoff(retry)
171+
try {
172+
await remove(getCacheDir(siteRoot))
173+
} catch {
174+
// in windows we might get "EBUSY" errors if LMDB failed to close, so this try/catch is
175+
// to prevent EBUSY errors from potentially hiding real import errors
176+
}
177+
await compileGatsbyFiles(siteRoot, retry + 1)
178+
return
179+
}
180+
} finally {
181+
worker.end()
131182
}
132183

133184
await exponentialBackoff(retry)
134185

135-
const bundles = bundleGraph.getBundles()
136-
137186
if (bundles.length === 0) return
138187

139188
let compiledTSFilesCount = 0
@@ -150,7 +199,7 @@ export async function compileGatsbyFiles(
150199
siteRoot,
151200
retries: RETRY_COUNT,
152201
compiledFileLocation: bundle.filePath,
153-
sourceFileLocation: bundle.getMainEntry()?.filePath,
202+
sourceFileLocation: bundle.mainEntryPath,
154203
},
155204
})
156205
} else if (retry > 0) {
@@ -165,9 +214,6 @@ export async function compileGatsbyFiles(
165214
)
166215
}
167216

168-
// sometimes parcel cache gets in weird state and we need to clear the cache
169-
await cacheClosePromise
170-
171217
try {
172218
await remove(getCacheDir(siteRoot))
173219
} catch {
@@ -179,7 +225,7 @@ export async function compileGatsbyFiles(
179225
return
180226
}
181227

182-
const mainEntry = bundle.getMainEntry()?.filePath
228+
const mainEntry = bundle.mainEntryPath
183229
// mainEntry won't exist for shared chunks
184230
if (mainEntry) {
185231
if (mainEntry.endsWith(`.ts`)) {

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy