1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
package httpcache
import (
"bufio"
hash "crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
)
var (
UserAgent string
ModifyResponse func(url string, entry CacheEntry, resp *http.Response) *http.Response
CheckRedirect func(req *http.Request, via []*http.Request) error
)
type CacheEntry string
var memCache = map[string]CacheEntry{}
type httpStatusError struct {
StatusCode int
Status string
}
// Is implements the interface for [errors.Is].
func (e *httpStatusError) Is(target error) bool {
switch target {
case os.ErrNotExist:
return e.StatusCode == http.StatusNotFound
default:
return false
}
}
// Error implements [error].
func (e *httpStatusError) Error() string {
return fmt.Sprintf("unexpected HTTP status: %v", e.Status)
}
type transport struct{}
func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
// Return an error for things that are the fault of things
// not-on-this-box. Panic for things that are the fault of
// this box.
// Initialize.
if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
panic(err)
}
// Calculate cache-key.
u := req.URL.String()
cacheKey := url.QueryEscape(u)
hdrKeys := make([]string, 0, len(req.Header))
for k := range req.Header {
switch k {
case "User-Agent":
case "Referer":
default:
hdrKeys = append(hdrKeys, http.CanonicalHeaderKey(k))
}
}
sort.Strings(hdrKeys)
for _, k := range hdrKeys {
cacheKey += "|" + url.QueryEscape(k) + ":" + url.QueryEscape(req.Header[k][0])
}
if len(cacheKey) >= 255 {
prefix := cacheKey[:255-(hash.Size*2)]
csum := hash.Sum([]byte(cacheKey))
suffix := hex.EncodeToString(csum[:])
cacheKey = prefix + suffix
}
cacheFile := filepath.Join(".http-cache", cacheKey)
// Check the mem cache.
if _, ok := memCache[cacheKey]; ok {
fmt.Printf("GET|CACHE|MEM %q...", u)
goto end
}
// Check the file cache.
if bs, err := os.ReadFile(cacheFile); err == nil {
str := string(bs)
if strings.HasPrefix(str, "HTTP/") || strings.HasPrefix(str, "CLIENT/") {
fmt.Printf("GET|CACHE|FILE %q...", u)
memCache[cacheKey] = CacheEntry(str)
goto end
}
}
// Do the request for real.
fmt.Printf("GET|NET %q...", u)
if resp, err := http.DefaultTransport.RoundTrip(req); err == nil {
var buf strings.Builder
if err := resp.Write(&buf); err != nil {
panic(err)
}
memCache[cacheKey] = CacheEntry(buf.String())
} else {
memCache[cacheKey] = CacheEntry("CLIENT/" + err.Error())
}
// Record the response to the file cache.
if err := os.WriteFile(cacheFile, []byte(memCache[cacheKey]), 0666); err != nil {
panic(err)
}
end:
// Turn the cache entry into an http.Response (or error)
var ret_resp *http.Response
var ret_err error
entry := memCache[cacheKey]
switch {
case strings.HasPrefix(string(entry), "HTTP/"):
var err error
ret_resp, err = http.ReadResponse(bufio.NewReader(strings.NewReader(string(entry))), nil)
if err != nil {
panic(fmt.Errorf("invalid cache entry: %v", err))
}
if ModifyResponse != nil {
ret_resp = ModifyResponse(u, entry, ret_resp)
}
case strings.HasPrefix(string(entry), "CLIENT/"):
ret_err = errors.New(string(entry)[len("CLIENT/"):])
default:
panic("invalid cache entry: invalid prefix")
}
// Return.
if ret_err != nil {
fmt.Printf(" err\n")
} else {
fmt.Printf(" http %v\n", ret_resp.StatusCode)
}
return ret_resp, ret_err
}
func Get(u string, hdr map[string]string) (string, error) {
if UserAgent == "" {
panic("main() must set the user agent string")
}
req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil {
panic(fmt.Errorf("should not happen: http.NewRequest: %v", err))
}
req.Header.Set("User-Agent", UserAgent)
for k, v := range hdr {
req.Header.Add(k, v)
}
client := &http.Client{
Transport: &transport{},
CheckRedirect: CheckRedirect,
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", &httpStatusError{StatusCode: resp.StatusCode, Status: resp.Status}
}
bs, err := io.ReadAll(resp.Body)
if err != nil {
panic(fmt.Errorf("should not happen: strings.Reader.Read: %v", err))
}
return string(bs), nil
}
func GetJSON(u string, hdr map[string]string, out any) error {
str, err := Get(u, hdr)
if err != nil {
return err
}
return json.Unmarshal([]byte(str), out)
}
func GetPaginatedJSON[T any](uStr string, hdr map[string]string, out *[]T, pageFn func(i int) url.Values) error {
u, err := url.Parse(uStr)
if err != nil {
return err
}
query := u.Query()
for i := 0; true; i++ {
pageParams := pageFn(i)
for k, v := range pageParams {
query[k] = v
}
u.RawQuery = query.Encode()
var resp []T
if err := GetJSON(u.String(), hdr, &resp); err != nil {
return err
}
fmt.Printf(" -> %d records\n", len(resp))
if len(resp) == 0 {
break
}
*out = append(*out, resp...)
}
return nil
}
|
