diff --git a/lib/get-client.js b/lib/get-client.js index f0725a65..66584aaf 100644 --- a/lib/get-client.js +++ b/lib/get-client.js @@ -1,35 +1,17 @@ -const {memoize, get} = require('lodash'); const {Octokit} = require('@octokit/rest'); -const pRetry = require('p-retry'); -const Bottleneck = require('bottleneck'); +const {throttling} = require('@octokit/plugin-throttling'); +const {retry} = require('@octokit/plugin-retry'); const urljoin = require('url-join'); const HttpProxyAgent = require('http-proxy-agent'); const HttpsProxyAgent = require('https-proxy-agent'); -const {RETRY_CONF, RATE_LIMITS, GLOBAL_RATE_LIMIT} = require('./definitions/rate-limit'); +const SemanticReleaseOctokit = Octokit.plugin(throttling, retry); -/** - * Http error status for which to not retry. - */ -const SKIP_RETRY_CODES = new Set([400, 401, 403]); - -/** - * Create or retrieve the throttler function for a given rate limit group. - * - * @param {Array} rate The rate limit group. - * @param {String} limit The rate limits per API endpoints. - * @param {Bottleneck} globalThrottler The global throttler. - * - * @return {Bottleneck} The throller function for the given rate limit group. - */ -const getThrottler = memoize((rate, globalThrottler) => - new Bottleneck({minTime: get(RATE_LIMITS, rate)}).chain(globalThrottler) -); +const {RETRY_CONF} = require('./definitions/rate-limit'); module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => { const baseUrl = githubUrl && urljoin(githubUrl, githubApiPathPrefix); - const globalThrottler = new Bottleneck({minTime: GLOBAL_RATE_LIMIT}); - const github = new Octokit({ + const github = new SemanticReleaseOctokit({ auth: `token ${githubToken}`, baseUrl, request: { @@ -39,24 +21,21 @@ module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => { : new HttpsProxyAgent(proxy) : undefined, }, - }); + throttle: { + onRateLimit: (retryAfter, options) => { + github.log.warn(`Request quota exhausted for request ${options.method} ${options.url}`); - github.hook.wrap('request', (request, options) => { - const access = options.method === 'GET' ? 'read' : 'write'; - const rateCategory = options.url.startsWith('/search') ? 'search' : 'core'; - const limitKey = [rateCategory, RATE_LIMITS[rateCategory][access] && access].filter(Boolean).join('.'); - - return pRetry(async () => { - try { - return await getThrottler(limitKey, globalThrottler).wrap(request)(options); - } catch (error) { - if (SKIP_RETRY_CODES.has(error.status)) { - throw new pRetry.AbortError(error); + if (options.request.retryCount <= RETRY_CONF.retries) { + github.log.debug(`Will retry after ${retryAfter}.`); + return true; } - throw error; - } - }, RETRY_CONF); + return false; + }, + onAbuseLimit: (retryAfter, options) => { + github.log.warn(`Abuse detected for request ${options.method} ${options.url}`); + }, + }, }); return github; diff --git a/package-lock.json b/package-lock.json index ed66058c..320a85e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -528,6 +528,24 @@ "deprecation": "^2.3.1" } }, + "@octokit/plugin-retry": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.7.tgz", + "integrity": "sha512-n08BPfVeKj5wnyH7IaOWnuKbx+e9rSJkhDHMJWXLPv61625uWjsN8G7sAW3zWm9n9vnS4friE7LL/XLcyGeG8Q==", + "requires": { + "@octokit/types": "^6.0.3", + "bottleneck": "^2.15.3" + } + }, + "@octokit/plugin-throttling": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.4.1.tgz", + "integrity": "sha512-qCQ+Z4AnL9OrXvV59EH3GzPxsB+WyqufoCjiCJXJxTbnt3W+leXbXw5vHrMp4NG9ltw00McFWIxIxNQAzLNoTA==", + "requires": { + "@octokit/types": "^6.0.1", + "bottleneck": "^2.15.3" + } + }, "@octokit/request": { "version": "5.4.15", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.15.tgz", @@ -782,7 +800,8 @@ "@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true }, "@typescript-eslint/eslint-plugin": { "version": "4.23.0", @@ -7769,6 +7788,7 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.5.0.tgz", "integrity": "sha512-5Hwh4aVQSu6BEP+w2zKlVXtFAaYQe1qWuVADSgoeVlLjwe/Q/AMSoRR4MDeaAfu8llT+YNbEijWu/YF3m6avkg==", + "dev": true, "requires": { "@types/retry": "^0.12.0", "retry": "^0.12.0" @@ -8475,7 +8495,8 @@ "retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" + "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", + "dev": true }, "reusify": { "version": "1.0.4", diff --git a/package.json b/package.json index a8cf5657..6b49d1a7 100644 --- a/package.json +++ b/package.json @@ -16,10 +16,11 @@ "Gregor Martynus (https://twitter.com/gr2m)" ], "dependencies": { + "@octokit/plugin-retry": "^3.0.0", + "@octokit/plugin-throttling": "^3.4.0", "@octokit/rest": "^18.0.0", "@semantic-release/error": "^2.2.0", "aggregate-error": "^3.0.0", - "bottleneck": "^2.18.1", "debug": "^4.0.0", "dir-glob": "^3.0.0", "fs-extra": "^10.0.0", @@ -30,7 +31,6 @@ "lodash": "^4.17.4", "mime": "^2.4.3", "p-filter": "^2.0.0", - "p-retry": "^4.0.0", "url-join": "^4.0.0" }, "devDependencies": { diff --git a/test/get-client.test.js b/test/get-client.test.js index ee005b2a..d9bceedf 100644 --- a/test/get-client.test.js +++ b/test/get-client.test.js @@ -4,12 +4,10 @@ const https = require('https'); const {promisify} = require('util'); const {readFile} = require('fs-extra'); const test = require('ava'); -const {inRange} = require('lodash'); -const {stub, spy} = require('sinon'); +const {spy} = require('sinon'); const proxyquire = require('proxyquire'); const Proxy = require('proxy'); const serverDestroy = require('server-destroy'); -const {Octokit} = require('@octokit/rest'); const rateLimit = require('./helpers/rate-limit'); const getClient = proxyquire('../lib/get-client', {'./definitions/rate-limit': rateLimit}); @@ -114,124 +112,3 @@ test.serial('Do not use a proxy if set to false', async (t) => { await promisify(server.destroy).bind(server)(); }); - -test('Use the global throttler for all endpoints', async (t) => { - const rate = 150; - - const octokit = new Octokit(); - octokit.hook.wrap('request', () => Date.now()); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': {RATE_LIMITS: {search: 1, core: 1}, GLOBAL_RATE_LIMIT: rate}, - })({githubToken: 'token'}); - - /* eslint-disable unicorn/prevent-abbreviations */ - - const a = await github.repos.createRelease(); - const b = await github.issues.createComment(); - const c = await github.repos.createRelease(); - const d = await github.issues.createComment(); - const e = await github.search.issuesAndPullRequests(); - const f = await github.search.issuesAndPullRequests(); - - // `issues.createComment` should be called `rate` ms after `repos.createRelease` - t.true(inRange(b - a, rate - 50, rate + 50)); - // `repos.createRelease` should be called `rate` ms after `issues.createComment` - t.true(inRange(c - b, rate - 50, rate + 50)); - // `issues.createComment` should be called `rate` ms after `repos.createRelease` - t.true(inRange(d - c, rate - 50, rate + 50)); - // `search.issuesAndPullRequests` should be called `rate` ms after `issues.createComment` - t.true(inRange(e - d, rate - 50, rate + 50)); - // `search.issuesAndPullRequests` should be called `rate` ms after `search.issuesAndPullRequests` - t.true(inRange(f - e, rate - 50, rate + 50)); - - /* eslint-enable unicorn/prevent-abbreviations */ -}); - -test('Use the same throttler for endpoints in the same rate limit group', async (t) => { - const searchRate = 300; - const coreRate = 150; - - const octokit = new Octokit(); - octokit.hook.wrap('request', () => Date.now()); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': {RATE_LIMITS: {search: searchRate, core: coreRate}, GLOBAL_RATE_LIMIT: 1}, - })({githubToken: 'token'}); - - /* eslint-disable unicorn/prevent-abbreviations */ - - const a = await github.repos.createRelease(); - const b = await github.issues.createComment(); - const c = await github.repos.createRelease(); - const d = await github.issues.createComment(); - const e = await github.search.issuesAndPullRequests(); - const f = await github.search.issuesAndPullRequests(); - - // `issues.createComment` should be called `coreRate` ms after `repos.createRelease` - t.true(inRange(b - a, coreRate - 50, coreRate + 50)); - // `repos.createRelease` should be called `coreRate` ms after `issues.createComment` - t.true(inRange(c - b, coreRate - 50, coreRate + 50)); - // `issues.createComment` should be called `coreRate` ms after `repos.createRelease` - t.true(inRange(d - c, coreRate - 50, coreRate + 50)); - - // The first search should be called immediately as it uses a different throttler - t.true(inRange(e - d, -50, 50)); - // The second search should be called only after `searchRate` ms - t.true(inRange(f - e, searchRate - 50, searchRate + 50)); - - /* eslint-enable unicorn/prevent-abbreviations */ -}); - -test('Use different throttler for read and write endpoints', async (t) => { - const writeRate = 300; - const readRate = 150; - - const octokit = new Octokit(); - octokit.hook.wrap('request', () => Date.now()); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': {RATE_LIMITS: {core: {write: writeRate, read: readRate}}, GLOBAL_RATE_LIMIT: 1}, - })({githubToken: 'token'}); - - const a = await github.repos.get(); - const b = await github.repos.get(); - const c = await github.repos.createRelease(); - const d = await github.repos.createRelease(); - - // `repos.get` should be called `readRate` ms after `repos.get` - t.true(inRange(b - a, readRate - 50, readRate + 50)); - // `repos.createRelease` should be called `coreRate` ms after `repos.createRelease` - t.true(inRange(d - c, writeRate - 50, writeRate + 50)); -}); - -test('Use the same throttler when retrying', async (t) => { - const coreRate = 200; - const request = stub().callsFake(async () => { - const err = new Error(); - err.time = Date.now(); - err.status = 404; - throw err; - }); - const octokit = new Octokit(); - octokit.hook.wrap('request', request); - const github = proxyquire('../lib/get-client', { - '@octokit/rest': {Octokit: stub().returns(octokit)}, - './definitions/rate-limit': { - RETRY_CONF: {retries: 3, factor: 1, minTimeout: 1}, - RATE_LIMITS: {core: coreRate}, - GLOBAL_RATE_LIMIT: 1, - }, - })({githubToken: 'token'}); - - await t.throwsAsync(github.repos.createRelease()); - const {time: a} = await t.throwsAsync(request.getCall(0).returnValue); - const {time: b} = await t.throwsAsync(request.getCall(1).returnValue); - const {time: c} = await t.throwsAsync(request.getCall(2).returnValue); - const {time: d} = await t.throwsAsync(request.getCall(3).returnValue); - - // Each retry should be done after `coreRate` ms - t.true(inRange(b - a, coreRate - 50, coreRate + 50)); - t.true(inRange(c - b, coreRate - 50, coreRate + 50)); - t.true(inRange(d - c, coreRate - 50, coreRate + 50)); -}); diff --git a/test/verify.test.js b/test/verify.test.js index f428e741..be8e1678 100644 --- a/test/verify.test.js +++ b/test/verify.test.js @@ -65,7 +65,11 @@ test.serial( await t.notThrowsAsync( verify( {proxy, assets, successComment, failTitle, failComment, labels}, - {env, options: {repositoryUrl: `git+https://othertesturl.com/${owner}/${repo}.git`}, logger: t.context.logger} + { + env, + options: {repositoryUrl: `git+https://othertesturl.com/${owner}/${repo}.git`}, + logger: t.context.logger, + } ) ); t.true(github.isDone()); @@ -440,7 +444,11 @@ test('Throw SemanticReleaseError for missing github token', async (t) => { const [error, ...errors] = await t.throwsAsync( verify( {}, - {env: {}, options: {repositoryUrl: 'https://github.com/semantic-release/github.git'}, logger: t.context.logger} + { + env: {}, + options: {repositoryUrl: 'https://github.com/semantic-release/github.git'}, + logger: t.context.logger, + } ) ); @@ -526,7 +534,7 @@ test.serial("Throw SemanticReleaseError if the repository doesn't exist", async const owner = 'test_user'; const repo = 'test_repo'; const env = {GH_TOKEN: 'github_token'}; - const github = authenticate(env).get(`/repos/${owner}/${repo}`).times(4).reply(404); + const github = authenticate(env).get(`/repos/${owner}/${repo}`).reply(404); const [error, ...errors] = await t.throwsAsync( verify({}, {env, options: {repositoryUrl: `https://github.com/${owner}/${repo}.git`}, logger: t.context.logger}) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy