From 3bb448ffd5d0c45a820dfd3dcd4fa9c69fadbe44 Mon Sep 17 00:00:00 2001 From: AerysNan Date: Thu, 10 Sep 2020 22:00:27 +0800 Subject: [PATCH 1/4] get tags from GraphQL API --- lib/config.js | 47 +++++++++++++++++++++++++++++ lib/plugins/company.js | 3 -- lib/plugins/leetcode.js | 67 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/lib/config.js b/lib/config.js index 82cfee17..8f39a25c 100644 --- a/lib/config.js +++ b/lib/config.js @@ -32,6 +32,52 @@ const DEFAULT_CONFIG = { 'swift', 'typescript' ], + tags: [ + 'array', + 'backtracking', + 'binary-indexed-tree', + 'binary-search', + 'binary-search-tree', + 'bit-manipulation', + 'brainteaser', + 'breadth-first-search', + 'depth-first-search', + 'design', + 'divide-and-conquer', + 'dynamic-programming', + 'geometry', + 'graph', + 'greedy', + 'hash-table', + 'heap', + 'line-sweep', + 'linked-list', + 'math', + 'memoization', + 'minimax', + 'ordered-map', + 'queue', + 'random', + 'recursion', + 'rejection-sampling', + 'reservoir-sampling', + 'segment-tree', + 'sliding-window', + 'sort', + 'stack', + 'string', + 'topological-sort', + 'tree', + 'trie', + 'two-pointers', + 'union-find', + + // TODO: these two tags below are included on leetcode.com but not on leetcode-cn.com + // Currently comment out for simplicity + + // 'rolling-hash', + // 'suffix-array', + ], urls: { // base urls base: 'https://leetcode.com', @@ -51,6 +97,7 @@ const DEFAULT_CONFIG = { linkedin_login_request: 'https://www.linkedin.com/login', linkedin_session_request: 'https://www.linkedin.com/checkpoint/lg/login-submit', // questions urls + tag: 'https://leetcode.com/tag/$tag/', problems: 'https://leetcode.com/api/problems/$category/', problem: 'https://leetcode.com/problems/$slug/description/', test: 'https://leetcode.com/problems/$slug/interpret_solution/', diff --git a/lib/plugins/company.js b/lib/plugins/company.js index 7959363b..52a641a4 100644 --- a/lib/plugins/company.js +++ b/lib/plugins/company.js @@ -1520,9 +1520,6 @@ plugin.getProblems = function(cb) { if (id in COMPONIES) { problem.companies = (problem.companies || []).concat(COMPONIES[id]); } - if (id in TAGS) { - problem.tags = (problem.tags || []).concat(TAGS[id]); - } }); return cb(null, problems); }); diff --git a/lib/plugins/leetcode.js b/lib/plugins/leetcode.js index c403582d..55b2f254 100644 --- a/lib/plugins/leetcode.js +++ b/lib/plugins/leetcode.js @@ -60,9 +60,9 @@ plugin.getProblems = function(cb) { const getCategory = function(category, queue, cb) { plugin.getCategoryProblems(category, function(e, _problems) { if (e) { - log.debug(category + ': failed to getProblems: ' + e.msg); + log.debug(category + ': failed to getCategory: ' + e.msg); } else { - log.debug(category + ': getProblems got ' + _problems.length + ' problems'); + log.debug(category + ': getCategory got ' + _problems.length + ' problems'); problems = problems.concat(_problems); } return cb(e); @@ -73,10 +73,68 @@ plugin.getProblems = function(cb) { const q = new Queue(config.sys.categories, {}, getCategory); q.run(null, function(e) { spin.stop(); - return cb(e, problems); + if (e) return cb(e); + plugin.attachTags(problems, cb); }); }; +plugin.attachTags = function(problems, cb) { + problems = new Map(problems.map(p => [p.id, p])); + const getTag = function (tag, queue, cb) { + plugin.getTagProblems(tag, function(e, _problems) { + if (e) { + log.debug(tag + ': failed to getTag: ' + e.msg); + } else if (!_problems) { + log.warn(tag + ': retrieve empty tag'); + } else { + log.debug(tag + ': getTag got ' + _problems.length + ' problems'); + _problems.forEach(function (p) { + let id = parseInt(p.questionId); + if (problems.has(id)) { + problems.get(id).tags.push(tag); + } + }) + } + return cb(e); + }); + }; + spin = h.spin("Downloading tags"); + const q = new Queue(config.sys.tags, {}, getTag); + q.run(null, function (e) { + spin.stop(); + problems = Array.from(problems.values()); + return cb(e, problems); + }); +} + +plugin.getTagProblems = function(tag, cb) { + log.debug('running leetcode.getTagProblems: ' + tag); + const opts = plugin.makeOpts(config.sys.urls.graphql); + opts.headers.Origin = config.sys.urls.base; + opts.headers.Referer = config.sys.urls.tag.replace('$tag', tag); + opts.json = true; + opts.body = { + query: [ + 'query getTopicTag($slug: String!) {', + ' topicTag(slug: $slug) {', + ' slug', + ' questions {', + ' questionId', + ' }', + ' }', + '}' + ].join('\n'), + operationName: 'getTopicTag', + variables: { slug: tag }, + }; + spin.text = 'Downloading tag ' + tag; + request(opts, function(e, resp, body) { + e = plugin.checkError(e, resp, 200); + if (e) return cb(e); + return cb(null, body.data.topicTag.questions); + }); +} + plugin.getCategoryProblems = function(category, cb) { log.debug('running leetcode.getCategoryProblems: ' + category); const opts = plugin.makeOpts(config.sys.urls.problems.replace('$category', category)); @@ -109,7 +167,8 @@ plugin.getCategoryProblems = function(category, cb) { percent: p.stat.total_acs * 100 / p.stat.total_submitted, level: h.levelToName(p.difficulty.level), starred: p.is_favor, - category: json.category_slug + category: json.category_slug, + tags: [] }; }); From 74d6fe6ed4ebe9e4a40f7a508b05a9f43511df7e Mon Sep 17 00:00:00 2001 From: AerysNan Date: Fri, 11 Sep 2020 10:49:32 +0800 Subject: [PATCH 2/4] make the problem list natural ordered --- lib/plugins/leetcode.js | 3 ++- package-lock.json | 5 +++++ package.json | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/plugins/leetcode.js b/lib/plugins/leetcode.js index 55b2f254..6cefc3b3 100644 --- a/lib/plugins/leetcode.js +++ b/lib/plugins/leetcode.js @@ -3,6 +3,7 @@ var util = require('util'); var _ = require('underscore'); var request = require('request'); +var { orderBy } = require('natural-orderby'); var prompt = require('prompt'); var config = require('../config'); @@ -102,7 +103,7 @@ plugin.attachTags = function(problems, cb) { const q = new Queue(config.sys.tags, {}, getTag); q.run(null, function (e) { spin.stop(); - problems = Array.from(problems.values()); + problems = orderBy(Array.from(problems.values()), [p => p.fid], ['asc']); return cb(e, problems); }); } diff --git a/package-lock.json b/package-lock.json index 607ea460..23028703 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2871,6 +2871,11 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "natural-orderby": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/natural-orderby/-/natural-orderby-2.0.3.tgz", + "integrity": "sha512-p7KTHxU0CUrcOXe62Zfrb5Z13nLvPhSWR/so3kFulUQU0sgUll2Z0LwpsLN351eOOD+hRGu/F1g+6xDfPeD++Q==" + }, "nconf": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/nconf/-/nconf-0.10.0.tgz", diff --git a/package.json b/package.json index 71aff1a8..ea1c6b76 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "he": "1.2.0", "mkdirp": "^1.0.4", "moment": "^2.20.1", + "natural-orderby": "^2.0.3", "nconf": "0.10.0", "ora": "3.0.0", "prompt": "1.0.0", From fee7eed5e74cd21eea1507922247274e9a99208d Mon Sep 17 00:00:00 2001 From: AerysNan Date: Fri, 11 Sep 2020 11:08:46 +0800 Subject: [PATCH 3/4] fix eslint --- lib/plugins/leetcode.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/plugins/leetcode.js b/lib/plugins/leetcode.js index 6cefc3b3..e402263c 100644 --- a/lib/plugins/leetcode.js +++ b/lib/plugins/leetcode.js @@ -99,7 +99,7 @@ plugin.attachTags = function(problems, cb) { return cb(e); }); }; - spin = h.spin("Downloading tags"); + spin = h.spin('Downloading tags'); const q = new Queue(config.sys.tags, {}, getTag); q.run(null, function (e) { spin.stop(); @@ -125,8 +125,8 @@ plugin.getTagProblems = function(tag, cb) { ' }', '}' ].join('\n'), - operationName: 'getTopicTag', - variables: { slug: tag }, + variables: {slug: tag}, + operationName: 'getTopicTag' }; spin.text = 'Downloading tag ' + tag; request(opts, function(e, resp, body) { From d0bdedad747d0393b1ac0402ef7f5e2668943fdb Mon Sep 17 00:00:00 2001 From: AerysNan Date: Fri, 11 Sep 2020 11:31:29 +0800 Subject: [PATCH 4/4] update test --- lib/plugins/leetcode.js | 65 ++++++++++++++++++----------------- test/mock/tags.json.20200911 | 1 + test/plugins/test_leetcode.js | 9 +++-- 3 files changed, 42 insertions(+), 33 deletions(-) create mode 100644 test/mock/tags.json.20200911 diff --git a/lib/plugins/leetcode.js b/lib/plugins/leetcode.js index e402263c..4afc2a3e 100644 --- a/lib/plugins/leetcode.js +++ b/lib/plugins/leetcode.js @@ -55,8 +55,7 @@ plugin.init = function() { config.app = 'leetcode'; }; -plugin.getProblems = function(cb) { - log.debug('running leetcode.getProblems'); +plugin.getProblemsWithoutTags = function(cb) { let problems = []; const getCategory = function(category, queue, cb) { plugin.getCategoryProblems(category, function(e, _problems) { @@ -74,37 +73,40 @@ plugin.getProblems = function(cb) { const q = new Queue(config.sys.categories, {}, getCategory); q.run(null, function(e) { spin.stop(); - if (e) return cb(e); - plugin.attachTags(problems, cb); + return cb(e, problems); }); }; -plugin.attachTags = function(problems, cb) { - problems = new Map(problems.map(p => [p.id, p])); - const getTag = function (tag, queue, cb) { - plugin.getTagProblems(tag, function(e, _problems) { - if (e) { - log.debug(tag + ': failed to getTag: ' + e.msg); - } else if (!_problems) { - log.warn(tag + ': retrieve empty tag'); - } else { - log.debug(tag + ': getTag got ' + _problems.length + ' problems'); - _problems.forEach(function (p) { - let id = parseInt(p.questionId); - if (problems.has(id)) { - problems.get(id).tags.push(tag); - } - }) - } - return cb(e); - }); - }; - spin = h.spin('Downloading tags'); - const q = new Queue(config.sys.tags, {}, getTag); - q.run(null, function (e) { - spin.stop(); - problems = orderBy(Array.from(problems.values()), [p => p.fid], ['asc']); - return cb(e, problems); +plugin.getProblems = function(cb) { + log.debug('running leetcode.getProblems'); + plugin.getProblemsWithoutTags(function(e, problems) { + if (e) return cb(e); + problems = new Map(problems.map(p => [p.id, p])); + const getTag = function (tag, queue, cb) { + plugin.getTagProblems(tag, function(e, _problems) { + if (e) { + log.debug(tag + ': failed to getTag: ' + JSON.stringify(e)); + } else if (!_problems) { + log.debug(tag + ': retrieve empty tag'); + } else { + log.debug(tag + ': getTag got ' + _problems.length + ' problems'); + _problems.forEach(function (p) { + let id = parseInt(p.questionId); + if (problems.has(id)) { + problems.get(id).tags.push(tag); + } + }) + } + return cb(e); + }); + }; + spin = h.spin('Downloading tags'); + const q = new Queue(config.sys.tags, {}, getTag); + q.run(null, function (e) { + spin.stop(); + problems = orderBy(Array.from(problems.values()), [p => p.fid], ['asc']); + return cb(e, problems); + }); }); } @@ -129,9 +131,10 @@ plugin.getTagProblems = function(tag, cb) { operationName: 'getTopicTag' }; spin.text = 'Downloading tag ' + tag; - request(opts, function(e, resp, body) { + request.post(opts, function(e, resp, body) { e = plugin.checkError(e, resp, 200); if (e) return cb(e); + if(!body || !body.data || !body.data.topicTag) return cb('receive invalid response body'); return cb(null, body.data.topicTag.questions); }); } diff --git a/test/mock/tags.json.20200911 b/test/mock/tags.json.20200911 new file mode 100644 index 00000000..b68e4b98 --- /dev/null +++ b/test/mock/tags.json.20200911 @@ -0,0 +1 @@ +{"data":{"topicTag":{"slug":"array","questions":[{"questionId":"1"},{"questionId":"4"},{"questionId":"11"},{"questionId":"15"},{"questionId":"16"},{"questionId":"18"},{"questionId":"26"},{"questionId":"27"},{"questionId":"31"},{"questionId":"33"},{"questionId":"34"},{"questionId":"35"},{"questionId":"39"},{"questionId":"40"},{"questionId":"41"},{"questionId":"42"},{"questionId":"45"},{"questionId":"48"},{"questionId":"53"},{"questionId":"54"},{"questionId":"55"},{"questionId":"56"},{"questionId":"57"},{"questionId":"59"},{"questionId":"62"},{"questionId":"63"},{"questionId":"64"},{"questionId":"66"},{"questionId":"73"},{"questionId":"74"},{"questionId":"75"},{"questionId":"78"},{"questionId":"79"},{"questionId":"80"},{"questionId":"81"},{"questionId":"84"},{"questionId":"85"},{"questionId":"88"},{"questionId":"90"},{"questionId":"105"},{"questionId":"106"},{"questionId":"118"},{"questionId":"119"},{"questionId":"120"},{"questionId":"121"},{"questionId":"122"},{"questionId":"123"},{"questionId":"126"},{"questionId":"128"},{"questionId":"152"},{"questionId":"153"},{"questionId":"154"},{"questionId":"162"},{"questionId":"163"},{"questionId":"167"},{"questionId":"169"},{"questionId":"189"},{"questionId":"209"},{"questionId":"216"},{"questionId":"217"},{"questionId":"219"},{"questionId":"228"},{"questionId":"229"},{"questionId":"238"},{"questionId":"243"},{"questionId":"245"},{"questionId":"259"},{"questionId":"268"},{"questionId":"277"},{"questionId":"280"},{"questionId":"283"},{"questionId":"287"},{"questionId":"289"},{"questionId":"370"},{"questionId":"380"},{"questionId":"381"},{"questionId":"414"},{"questionId":"442"},{"questionId":"448"},{"questionId":"457"},{"questionId":"485"},{"questionId":"495"},{"questionId":"531"},{"questionId":"532"},{"questionId":"533"},{"questionId":"548"},{"questionId":"560"},{"questionId":"561"},{"questionId":"562"},{"questionId":"565"},{"questionId":"566"},{"questionId":"581"},{"questionId":"605"},{"questionId":"611"},{"questionId":"621"},{"questionId":"624"},{"questionId":"628"},{"questionId":"643"},{"questionId":"644"},{"questionId":"661"},{"questionId":"665"},{"questionId":"667"},{"questionId":"670"},{"questionId":"674"},{"questionId":"689"},{"questionId":"695"},{"questionId":"697"},{"questionId":"713"},{"questionId":"714"},{"questionId":"717"},{"questionId":"718"},{"questionId":"719"},{"questionId":"723"},{"questionId":"724"},{"questionId":"729"},{"questionId":"747"},{"questionId":"748"},{"questionId":"756"},{"questionId":"777"},{"questionId":"779"},{"questionId":"780"},{"questionId":"790"},{"questionId":"798"},{"questionId":"808"},{"questionId":"811"},{"questionId":"852"},{"questionId":"857"},{"questionId":"861"},{"questionId":"864"},{"questionId":"870"},{"questionId":"879"},{"questionId":"898"},{"questionId":"901"},{"questionId":"905"},{"questionId":"924"},{"questionId":"927"},{"questionId":"932"},{"questionId":"936"},{"questionId":"941"},{"questionId":"943"},{"questionId":"950"},{"questionId":"951"},{"questionId":"954"},{"questionId":"958"},{"questionId":"962"},{"questionId":"978"},{"questionId":"982"},{"questionId":"987"},{"questionId":"991"},{"questionId":"1002"},{"questionId":"1009"},{"questionId":"1013"},{"questionId":"1016"},{"questionId":"1019"},{"questionId":"1020"},{"questionId":"1027"},{"questionId":"1031"},{"questionId":"1041"},{"questionId":"1044"},{"questionId":"1049"},{"questionId":"1055"},{"questionId":"1056"},{"questionId":"1062"},{"questionId":"1063"},{"questionId":"1066"},{"questionId":"1071"},{"questionId":"1074"},{"questionId":"1082"},{"questionId":"1083"},{"questionId":"1096"},{"questionId":"1098"},{"questionId":"1102"},{"questionId":"1105"},{"questionId":"1107"},{"questionId":"1108"},{"questionId":"1112"},{"questionId":"1113"},{"questionId":"1137"},{"questionId":"1138"},{"questionId":"1139"},{"questionId":"1145"},{"questionId":"1168"},{"questionId":"1175"},{"questionId":"1206"},{"questionId":"1217"},{"questionId":"1221"},{"questionId":"1227"},{"questionId":"1231"},{"questionId":"1232"},{"questionId":"1241"},{"questionId":"1247"},{"questionId":"1249"},{"questionId":"1253"},{"questionId":"1255"},{"questionId":"1256"},{"questionId":"1262"},{"questionId":"1272"},{"questionId":"1273"},{"questionId":"1280"},{"questionId":"1281"},{"questionId":"1287"},{"questionId":"1289"},{"questionId":"1293"},{"questionId":"1306"},{"questionId":"1308"},{"questionId":"1321"},{"questionId":"1329"},{"questionId":"1342"},{"questionId":"1345"},{"questionId":"1349"},{"questionId":"1350"},{"questionId":"1374"},{"questionId":"1378"},{"questionId":"1386"},{"questionId":"1391"},{"questionId":"1395"},{"questionId":"1396"},{"questionId":"1400"},{"questionId":"1402"},{"questionId":"1413"},{"questionId":"1421"},{"questionId":"1422"},{"questionId":"1426"},{"questionId":"1445"},{"questionId":"1455"},{"questionId":"1463"},{"questionId":"1464"},{"questionId":"1468"},{"questionId":"1476"},{"questionId":"1477"},{"questionId":"1482"},{"questionId":"1483"},{"questionId":"1486"},{"questionId":"1487"},{"questionId":"1491"},{"questionId":"1496"},{"questionId":"1500"},{"questionId":"1505"},{"questionId":"1510"},{"questionId":"1511"},{"questionId":"1514"},{"questionId":"1515"},{"questionId":"1525"},{"questionId":"1528"},{"questionId":"1538"},{"questionId":"1539"},{"questionId":"1548"},{"questionId":"1549"},{"questionId":"1553"},{"questionId":"1556"},{"questionId":"1560"},{"questionId":"1570"},{"questionId":"1572"},{"questionId":"1574"},{"questionId":"1575"},{"questionId":"1580"},{"questionId":"1581"},{"questionId":"1584"},{"questionId":"1586"},{"questionId":"1603"},{"questionId":"1604"},{"questionId":"1605"},{"questionId":"1610"},{"questionId":"1612"},{"questionId":"1615"},{"questionId":"1616"},{"questionId":"1620"},{"questionId":"1622"},{"questionId":"1626"},{"questionId":"1627"},{"questionId":"1631"},{"questionId":"1635"},{"questionId":"1640"},{"questionId":"1646"},{"questionId":"1656"},{"questionId":"1657"},{"questionId":"1675"},{"questionId":"1677"},{"questionId":"1679"},{"questionId":"1682"},{"questionId":"1689"},{"questionId":"1713"}]}}} \ No newline at end of file diff --git a/test/plugins/test_leetcode.js b/test/plugins/test_leetcode.js index 39a1431e..3f896efb 100644 --- a/test/plugins/test_leetcode.js +++ b/test/plugins/test_leetcode.js @@ -124,10 +124,15 @@ describe('plugin:leetcode', function() { nock('https://leetcode.com') .get('/api/problems/concurrency/') .replyWithFile(200, './test/mock/problems.json.20160911'); - + + nock('https://leetcode.com') + .post('/graphql') + .times(config.sys.tags.length) + .replyWithFile(200, './test/mock/tags.json.20200911'); + plugin.getProblems(function(e, problems) { assert.equal(e, null); - assert.equal(problems.length, 377 * 4); + assert.equal(problems.length, 377); done(); }); }); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy