1 files changed, 36 insertions, 0 deletions
diff --git a/cmd/generate/httpcache.go b/cmd/generate/httpcache.go
index cff0c7c..04762e3 100644
--- a/cmd/generate/httpcache.go
+++ b/cmd/generate/httpcache.go
@@ -14,6 +14,7 @@ var httpCache = map[string]string{}
 
 func httpGet(u string) (string, error) {
 	if cache, ok := httpCache[u]; ok {
+		fmt.Printf("CACHE-GET %q\n", u)
 		return cache, nil
 	}
 	if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
@@ -22,6 +23,7 @@ func httpGet(u string) (string, error) {
 	cacheFile := filepath.Join(".http-cache", url.QueryEscape(u))
 	if bs, err := os.ReadFile(cacheFile); err == nil {
 		httpCache[u] = string(bs)
+		fmt.Printf("CACHE-GET %q\n", u)
 		return httpCache[u], nil
 	} else if !os.IsNotExist(err) {
 		return "", err
@@ -57,3 +59,37 @@ func httpGetJSON(u string, out any) error {
 	}
 	return json.Unmarshal([]byte(str), out)
 }
+
+func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error {
+	u, err := url.Parse(uStr)
+	if err != nil {
+		return err
+	}
+	query := u.Query()
+
+	for i := 0; true; i++ {
+		pageParams := pageFn(i)
+		for k, v := range pageParams {
+			query[k] = v
+		}
+
+		u.RawQuery = query.Encode()
+		var resp []T
+		if err := httpGetJSON(u.String(), &resp); err != nil {
+			return err
+		}
+		fmt.Printf(" -> %d records\n", len(resp))
+		if len(resp) == 0 {
+			break
+		}
+		*out = append(*out, resp...)
+	}
+
+	return nil
+}
+
+func githubPagination(i int) url.Values {
+	params := make(url.Values)
+	params.Set("page", fmt.Sprintf("%v", i+1))
+	return params
+}