Pagination Support (Now you can fetch all the things!)

azenla · azenla · commit d83d5b9ed334 · 2014-08-02T12:49:56.000-04:00
diff --git a/lib/src/client/github.dart b/lib/src/client/github.dart
@@ -77,14 +77,20 @@ class GitHub {
   /**
    * Fetches the repositories of the user specified by [user].
    */
-  Future<List<Repository>> userRepositories(String user, {String type: "owner", int limit: 5000, String sort: "full_name", String direction: "asc"}) {
+  Future<List<Repository>> userRepositories(String user, {String type: "owner", int limit, String sort: "full_name", String direction: "asc"}) {
     var params = {
-      "per_page": limit.toString(),
       "sort": sort,
       "direction": direction
     };
-    return getJSON("/users/${user}/repos", params: params).then((List json) {
-      return new List.from(json.map((it) => Repository.fromJSON(this, it)));
+    
+    var pages = limit != null ? (limit / 30).ceil() : null;
+    
+    return new PaginationHelper(this).fetch("GET", "/users/${user}/repos", pages: pages, params: params).then((List<http.Response> responses) {
+      var list = <dynamic>[];
+      for (var response in responses) {
+        list.addAll(JSON.decode(response.body));
+      }
+      return new List.from(list.map((it) => Repository.fromJSON(this, it)));
     });
   }
 
@@ -115,11 +121,10 @@ class GitHub {
    * 
    * [name] is the organization name.
    * [limit] is the maximum number of teams to provide.
-   * Currently the highest you can go is 30.
    */
   Future<List<Team>> teams(String name, [int limit]) {
     var group = new FutureGroup<Team>();
-    getJSON("/orgs/${name}/teams").then((teams) {
+    getJSON("/orgs/${name}/teams?per_page=${limit}").then((teams) {
       for (var team in teams) {
         group.add(getJSON(team['url'], convert: Team.fromJSON, statusCode: 200, fail: (http.Response response) {
           if (response.statusCode == 404) {
@@ -178,12 +183,17 @@ class GitHub {
    * Gets a Repositories Releases.
    * 
    * [slug] is the repository to fetch releases from.
-   * [limit] is the maximum number of pages.
-   * Currently the maximum limit is 100.
+   * [limit] is the maximum number of releases to show.
    */
-  Future<List<Release>> releases(RepositorySlug slug, [int limit = 30]) {
-    return getJSON("/repos/${slug.fullName}/releases", params: { "per_page": limit }).then((releases) {
-      return copyOf(releases.map((it) => Release.fromJSON(this, it)));
+  Future<List<Release>> releases(RepositorySlug slug, {int limit}) {
+    var pages = limit != null ? (limit / 30).ceil() : null;
+    
+    return new PaginationHelper(this).fetch("GET", "/repos/${slug.fullName}/releases", pages: pages, params: {}).then((List<http.Response> responses) {
+      var list = <dynamic>[];
+      for (var response in responses) {
+        list.addAll(JSON.decode(response.body));
+      }
+      return new List.from(list.map((it) => Repository.fromJSON(this, it)));
     });
   }
   
@@ -360,3 +370,52 @@ class GitHub {
     }
   }
 }
+
+class PaginationHelper {
+  final GitHub github;
+  final List<http.Response> responses;
+  final Completer<List<http.Response>> completer;
+  
+  PaginationHelper(this.github) : responses = [], completer = new Completer<List<http.Response>>();
+  
+  Future<List<http.Response>> fetch(String method, String path, {int pages, Map<String, String> headers, Map<String, dynamic> params, String body}) {
+    Future<http.Response> actualFetch(String realPath) {
+      return github.request(method, realPath, headers: headers, params: params, body: body);
+    }
+    
+    void done() => completer.complete(responses);
+    
+    var count = 0;
+    
+    var handleResponse;
+    handleResponse = (http.Response response) {
+      count++;
+      responses.add(response);
+      
+      if (!response.headers.containsKey("link")) {
+        done();
+        return;
+      }
+      
+      var info = parseLinkHeader(response.headers['link']);
+      
+      if (!info.containsKey("next")) {
+        done();
+        return;
+      }
+      
+      if (pages != null && count == pages) {
+        done();
+        return;
+      }
+      
+      var nextUrl = info['next'];
+      
+      actualFetch(nextUrl).then(handleResponse);
+    };
+    
+    actualFetch(path).then(handleResponse);
+    
+    return completer.future;
+  }
+}
diff --git a/lib/src/client/repo.dart b/lib/src/client/repo.dart
@@ -98,13 +98,13 @@ class Repository {
   
   RepositorySlug get slug => new RepositorySlug(owner.login, name);
   
-  Future<List<Issue>> issues([int limit = 30]) => github.getJSON("/repos/${fullName}/issues", params: { "per_page": limit }).then((json) {
+  Future<List<Issue>> issues({int limit: 30}) => github.getJSON("/repos/${fullName}/issues", params: { "per_page": limit }).then((json) {
     return json.map((it) => Issue.fromJSON(github, it));
   });
   
   Future<List<Commit>> get commits => github.getJSON("/repos/${fullName}/commits", convert: (github, it) => it.map((i) => Commit.fromJSON(github, i)));
   
-  Future<List<ContributorStatistics>> contributorStatistics([int limit = 30]) {
+  Future<List<ContributorStatistics>> contributorStatistics({int limit: 30}) {
     var completer = new Completer<List<ContributorStatistics>>();
     var path = "/repos/${fullName}/stats/contributors";
     var handle;
@@ -122,13 +122,13 @@ class Repository {
     return completer.future;
   }
   
-  Future<List<Repository>> forks([int limit = 30]) {
+  Future<List<Repository>> forks({int limit: 30}) {
     return github.getJSON("/repos/${fullName}/forks", params: { "per_page": limit }).then((forks) {
       return copyOf(forks.map((it) => Repository.fromJSON(github, it)));
     });
   }
   
-  Future<List<PullRequest>> pullRequests([int limit = 30]) {
+  Future<List<PullRequest>> pullRequests({int limit: 30}) {
     return github.getJSON("/repos/${fullName}/pulls", params: { "per_page": limit }).then((List<Map> pulls) {
       return copyOf(pulls.map((it) => PullRequest.fromJSON(github, it)));
     });
@@ -138,13 +138,13 @@ class Repository {
     return github.getJSON("/repos/${fullName}/pages", convert: RepositoryPages.fromJSON);
   }
   
-  Future<List<Hook>> hooks([int limit = 30]) {
+  Future<List<Hook>> hooks({int limit: 30}) {
     return github.getJSON("/repos/${fullName}/hooks", params: { "per_page": limit }).then((hooks) {
       return copyOf(hooks.map((it) => Hook.fromJSON(github, fullName, it)));
     });
   }
   
-  Future<List<Release>> releases([int limit = 30]) => github.releases(slug, limit);
+  Future<List<Release>> releases({int limit}) => github.releases(slug, limit: limit);
   
   Future<Release> release(int id) => github.release(slug, id);
   
diff --git a/lib/src/client/util.dart b/lib/src/client/util.dart
@@ -82,6 +82,23 @@ void putValue(String name, dynamic value, Map map) {
   }
 }
 
+Map<String, String> parseLinkHeader(String input) {
+  var out = {};
+  var parts = input.split(", ");
+  for (var part in parts) {
+    if (part[0] != "<") {
+      throw new FormatException("Invalid Link Header");
+    }
+    var kv = part.split("; ");
+    var url = kv[0].substring(1);
+    url = url.substring(0, url.length - 1);
+    var key = kv[1];
+    key = key.replaceAll('"', "").substring(4);
+    out[key] = url;
+  }
+  return out;
+}
+
 String fullNameFromRepoApiUrl(String url) {
   var split = url.split("/");
   return split[4] + "/" + split[5];
diff --git a/test/link_header.dart b/test/link_header.dart
@@ -0,0 +1,6 @@
+import 'package:github/src/client/util.dart';
+
+void main() {
+  var it = parseLinkHeader('<https://api.github.com/user/1342004/repos?sort=stars&page=2>; rel="next", <https://api.github.com/user/1342004/repos?sort=stars&page=7>; rel="last"');
+  print(it);
+}
