package main import ( "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "sort" ) type httpCacheEntry struct { Body string Err error } var httpCache = map[string]httpCacheEntry{} type httpStatusError struct { StatusCode int Status string } // Is implements the interface for [errors.Is]. func (e *httpStatusError) Is(target error) bool { switch target { case os.ErrNotExist: return e.StatusCode == http.StatusNotFound default: return false } } // Error implements [error]. func (e *httpStatusError) Error() string { return fmt.Sprintf("unexpected HTTP status: %v", e.Status) } func httpGet(u string, hdr map[string]string) (string, error) { cacheKey := url.QueryEscape(u) hdrKeys := make([]string, 0, len(hdr)) for k := range hdr { hdrKeys = append(hdrKeys, http.CanonicalHeaderKey(k)) } sort.Strings(hdrKeys) for _, k := range hdrKeys { cacheKey += "|" + url.QueryEscape(k) + ":" + url.QueryEscape(hdr[k]) } if cache, ok := httpCache[cacheKey]; ok { fmt.Printf("CACHE-GET %q\n", u) return cache.Body, cache.Err } if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) { return "", err } cacheFile := filepath.Join(".http-cache", cacheKey) if bs, err := os.ReadFile(cacheFile); err == nil { fmt.Printf("CACHE-GET %q\n", u) httpCache[cacheKey] = httpCacheEntry{Body: string(bs)} return httpCache[cacheKey].Body, nil } else if !os.IsNotExist(err) { httpCache[cacheKey] = httpCacheEntry{Err: err} return "", err } fmt.Printf("GET %q...", u) req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { httpCache[cacheKey] = httpCacheEntry{Err: err} return "", err } for k, v := range hdr { req.Header.Add(k, v) } resp, err := http.DefaultClient.Do(req) if err != nil { fmt.Printf(" err\n") httpCache[cacheKey] = httpCacheEntry{Err: err} return "", err } if resp.StatusCode != http.StatusOK { fmt.Printf(" err\n") httpCache[cacheKey] = httpCacheEntry{Err: err} return "", &httpStatusError{StatusCode: resp.StatusCode, Status: resp.Status} } bs, err := io.ReadAll(resp.Body) if err != nil { fmt.Printf(" err\n") httpCache[cacheKey] = httpCacheEntry{Err: err} httpCache[cacheKey] = httpCacheEntry{Err: err} return "", err } fmt.Printf(" ok\n") if err := os.WriteFile(cacheFile, bs, 0666); err != nil { return "", err } httpCache[cacheKey] = httpCacheEntry{Body: string(bs)} return httpCache[cacheKey].Body, nil } func httpGetJSON(u string, hdr map[string]string, out any) error { str, err := httpGet(u, hdr) if err != nil { return err } return json.Unmarshal([]byte(str), out) } func httpGetPaginatedJSON[T any](uStr string, hdr map[string]string, out *[]T, pageFn func(i int) url.Values) error { u, err := url.Parse(uStr) if err != nil { return err } query := u.Query() for i := 0; true; i++ { pageParams := pageFn(i) for k, v := range pageParams { query[k] = v } u.RawQuery = query.Encode() var resp []T if err := httpGetJSON(u.String(), hdr, &resp); err != nil { return err } fmt.Printf(" -> %d records\n", len(resp)) if len(resp) == 0 { break } *out = append(*out, resp...) } return nil }