1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
)
var httpCache = map[string]string{}
func httpGet(u string) (string, error) {
if cache, ok := httpCache[u]; ok {
fmt.Printf("CACHE-GET %q\n", u)
return cache, nil
}
if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
return "", err
}
cacheFile := filepath.Join(".http-cache", url.QueryEscape(u))
if bs, err := os.ReadFile(cacheFile); err == nil {
httpCache[u] = string(bs)
fmt.Printf("CACHE-GET %q\n", u)
return httpCache[u], nil
} else if !os.IsNotExist(err) {
return "", err
}
fmt.Printf("GET %q...", u)
resp, err := http.Get(u)
if err != nil {
fmt.Printf(" err\n")
return "", err
}
if resp.StatusCode != http.StatusOK {
fmt.Printf(" err\n")
return "", fmt.Errorf("unexpected HTTP status: %v", resp.Status)
}
bs, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Printf(" err\n")
return "", err
}
fmt.Printf(" ok\n")
if err := os.WriteFile(cacheFile, bs, 0666); err != nil {
return "", err
}
httpCache[u] = string(bs)
return httpCache[u], nil
}
func httpGetJSON(u string, out any) error {
str, err := httpGet(u)
if err != nil {
return err
}
return json.Unmarshal([]byte(str), out)
}
func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error {
u, err := url.Parse(uStr)
if err != nil {
return err
}
query := u.Query()
for i := 0; true; i++ {
pageParams := pageFn(i)
for k, v := range pageParams {
query[k] = v
}
u.RawQuery = query.Encode()
var resp []T
if err := httpGetJSON(u.String(), &resp); err != nil {
return err
}
fmt.Printf(" -> %d records\n", len(resp))
if len(resp) == 0 {
break
}
*out = append(*out, resp...)
}
return nil
}
func githubPagination(i int) url.Values {
params := make(url.Values)
params.Set("page", fmt.Sprintf("%v", i+1))
return params
}
|
