package main import ( "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path/filepath" ) var httpCache = map[string]string{} func httpGet(u string) (string, error) { if cache, ok := httpCache[u]; ok { fmt.Printf("CACHE-GET %q\n", u) return cache, nil } if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) { return "", err } cacheFile := filepath.Join(".http-cache", url.QueryEscape(u)) if bs, err := os.ReadFile(cacheFile); err == nil { httpCache[u] = string(bs) fmt.Printf("CACHE-GET %q\n", u) return httpCache[u], nil } else if !os.IsNotExist(err) { return "", err } fmt.Printf("GET %q...", u) resp, err := http.Get(u) if err != nil { fmt.Printf(" err\n") return "", err } if resp.StatusCode != http.StatusOK { fmt.Printf(" err\n") return "", fmt.Errorf("unexpected HTTP status: %v", resp.Status) } bs, err := io.ReadAll(resp.Body) if err != nil { fmt.Printf(" err\n") return "", err } fmt.Printf(" ok\n") if err := os.WriteFile(cacheFile, bs, 0666); err != nil { return "", err } httpCache[u] = string(bs) return httpCache[u], nil } func httpGetJSON(u string, out any) error { str, err := httpGet(u) if err != nil { return err } return json.Unmarshal([]byte(str), out) } func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error { u, err := url.Parse(uStr) if err != nil { return err } query := u.Query() for i := 0; true; i++ { pageParams := pageFn(i) for k, v := range pageParams { query[k] = v } u.RawQuery = query.Encode() var resp []T if err := httpGetJSON(u.String(), &resp); err != nil { return err } fmt.Printf(" -> %d records\n", len(resp)) if len(resp) == 0 { break } *out = append(*out, resp...) } return nil } func githubPagination(i int) url.Values { params := make(url.Values) params.Set("page", fmt.Sprintf("%v", i+1)) return params }