1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
)
var httpCache = map[string]string{}
func httpGet(u string, hdr map[string]string) (string, error) {
cacheKey := url.QueryEscape(u)
hdrKeys := make([]string, 0, len(hdr))
for k := range hdr {
hdrKeys = append(hdrKeys, http.CanonicalHeaderKey(k))
}
sort.Strings(hdrKeys)
for _, k := range hdrKeys {
cacheKey += "|" + url.QueryEscape(k) + ":" + url.QueryEscape(hdr[k])
}
if cache, ok := httpCache[cacheKey]; ok {
fmt.Printf("CACHE-GET %q\n", u)
return cache, nil
}
if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
return "", err
}
cacheFile := filepath.Join(".http-cache", cacheKey)
if bs, err := os.ReadFile(cacheFile); err == nil {
httpCache[cacheKey] = string(bs)
fmt.Printf("CACHE-GET %q\n", u)
return httpCache[cacheKey], nil
} else if !os.IsNotExist(err) {
return "", err
}
fmt.Printf("GET %q...", u)
req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil {
return "", err
}
for k, v := range hdr {
req.Header.Add(k, v)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Printf(" err\n")
return "", err
}
if resp.StatusCode != http.StatusOK {
fmt.Printf(" err\n")
return "", fmt.Errorf("unexpected HTTP status: %v", resp.Status)
}
bs, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Printf(" err\n")
return "", err
}
fmt.Printf(" ok\n")
if err := os.WriteFile(cacheFile, bs, 0666); err != nil {
return "", err
}
httpCache[cacheKey] = string(bs)
return httpCache[cacheKey], nil
}
func httpGetJSON(u string, hdr map[string]string, out any) error {
str, err := httpGet(u, hdr)
if err != nil {
return err
}
return json.Unmarshal([]byte(str), out)
}
func httpGetPaginatedJSON[T any](uStr string, hdr map[string]string, out *[]T, pageFn func(i int) url.Values) error {
u, err := url.Parse(uStr)
if err != nil {
return err
}
query := u.Query()
for i := 0; true; i++ {
pageParams := pageFn(i)
for k, v := range pageParams {
query[k] = v
}
u.RawQuery = query.Encode()
var resp []T
if err := httpGetJSON(u.String(), hdr, &resp); err != nil {
return err
}
fmt.Printf(" -> %d records\n", len(resp))
if len(resp) == 0 {
break
}
*out = append(*out, resp...)
}
return nil
}
|
