cmd/generate: Allow specifying HTTP headers

4 files changed, 46 insertions, 28 deletions

diff --git a/cmd/generate/gerrit.go b/cmd/generate/gerrit.go
index c8837fc..d2e9b8b 100644
--- a/cmd/generate/gerrit.go
+++ b/cmd/generate/gerrit.go
@@ -10,8 +10,8 @@ import (
 
 // httpGetGerritJSON is like [httpGetJSON], but
 // https://gerrit-review.googlesource.com/Documentation/rest-api.html#output
-func httpGetGerritJSON(u string, out any) error {
-	str, err := httpGet(u)
+func httpGetGerritJSON(u string, hdr map[string]string, out any) error {
+	str, err := httpGet(u, hdr)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/generate/httpcache.go b/cmd/generate/httpcache.go
index 04762e3..08153d1 100644
--- a/cmd/generate/httpcache.go
+++ b/cmd/generate/httpcache.go
@@ -8,29 +8,47 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"sort"
 )
 
 var httpCache = map[string]string{}
 
-func httpGet(u string) (string, error) {
-	if cache, ok := httpCache[u]; ok {
+func httpGet(u string, hdr map[string]string) (string, error) {
+	cacheKey := url.QueryEscape(u)
+	hdrKeys := make([]string, 0, len(hdr))
+	for k := range hdr {
+		hdrKeys = append(hdrKeys, http.CanonicalHeaderKey(k))
+	}
+	sort.Strings(hdrKeys)
+	for _, k := range hdrKeys {
+		cacheKey += "|" + url.QueryEscape(k) + ":" + url.QueryEscape(hdr[k])
+	}
+
+	if cache, ok := httpCache[cacheKey]; ok {
 		fmt.Printf("CACHE-GET %q\n", u)
 		return cache, nil
 	}
 	if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
 		return "", err
 	}
-	cacheFile := filepath.Join(".http-cache", url.QueryEscape(u))
+	cacheFile := filepath.Join(".http-cache", cacheKey)
 	if bs, err := os.ReadFile(cacheFile); err == nil {
-		httpCache[u] = string(bs)
+		httpCache[cacheKey] = string(bs)
 		fmt.Printf("CACHE-GET %q\n", u)
-		return httpCache[u], nil
+		return httpCache[cacheKey], nil
 	} else if !os.IsNotExist(err) {
 		return "", err
 	}
 
 	fmt.Printf("GET %q...", u)
-	resp, err := http.Get(u)
+	req, err := http.NewRequest(http.MethodGet, u, nil)
+	if err != nil {
+		return "", err
+	}
+	for k, v := range hdr {
+		req.Header.Add(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		fmt.Printf(" err\n")
 		return "", err
@@ -48,19 +66,19 @@ func httpGet(u string) (string, error) {
 	if err := os.WriteFile(cacheFile, bs, 0666); err != nil {
 		return "", err
 	}
-	httpCache[u] = string(bs)
-	return httpCache[u], nil
+	httpCache[cacheKey] = string(bs)
+	return httpCache[cacheKey], nil
 }
 
-func httpGetJSON(u string, out any) error {
-	str, err := httpGet(u)
+func httpGetJSON(u string, hdr map[string]string, out any) error {
+	str, err := httpGet(u, hdr)
 	if err != nil {
 		return err
 	}
 	return json.Unmarshal([]byte(str), out)
 }
 
-func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error {
+func httpGetPaginatedJSON[T any](uStr string, hdr map[string]string, out *[]T, pageFn func(i int) url.Values) error {
 	u, err := url.Parse(uStr)
 	if err != nil {
 		return err
@@ -75,7 +93,7 @@ func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.V
 
 		u.RawQuery = query.Encode()
 		var resp []T
-		if err := httpGetJSON(u.String(), &resp); err != nil {
+		if err := httpGetJSON(u.String(), hdr, &resp); err != nil {
 			return err
 		}
 		fmt.Printf(" -> %d records\n", len(resp))
diff --git a/cmd/generate/src_contribs.go b/cmd/generate/src_contribs.go
index eb9d9ad..0b5acb5 100644
--- a/cmd/generate/src_contribs.go
+++ b/cmd/generate/src_contribs.go
@@ -124,7 +124,7 @@ func (c Contribution) fetchStatus() (string, error) {
 			Merged         bool   `json:"merged"`
 			MergeCommitSha string `json:"merge_commit_sha"`
 		}
-		if err := httpGetJSON(urlStr, &obj); err != nil {
+		if err := httpGetJSON(urlStr, nil, &obj); err != nil {
 			return "", err
 		}
 		ret := obj.State
@@ -154,7 +154,7 @@ func (c Contribution) fetchStatus() (string, error) {
 			MergeCommitSha  string `json:"merge_commit_sha"`
 			SquashCommitSha string `json:"squash_commit_sha"`
 		}
-		if err := httpGetJSON(urlStr, &obj); err != nil {
+		if err := httpGetJSON(urlStr, nil, &obj); err != nil {
 			return "", err
 		}
 
@@ -224,7 +224,7 @@ func (c Contribution) fetchSubmittedAt() (time.Time, error) {
 		var obj struct {
 			CreatedAt time.Time `json:"created_at"`
 		}
-		if err := httpGetJSON(urlStr, &obj); err != nil {
+		if err := httpGetJSON(urlStr, nil, &obj); err != nil {
 			return time.Time{}, err
 		}
 		return obj.CreatedAt, nil
@@ -239,13 +239,13 @@ func (c Contribution) fetchSubmittedAt() (time.Time, error) {
 		var obj struct {
 			CreatedAt time.Time `json:"created_at"`
 		}
-		if err := httpGetJSON(urlStr, &obj); err != nil {
+		if err := httpGetJSON(urlStr, nil, &obj); err != nil {
 			return time.Time{}, err
 		}
 		return obj.CreatedAt, nil
 	}
 	if strings.Contains(c.URLs[0], "/pipermail/") {
-		htmlStr, err := httpGet(c.URLs[0])
+		htmlStr, err := httpGet(c.URLs[0], nil)
 		if err != nil {
 			return time.Time{}, err
 		}
@@ -277,7 +277,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 					Date GerritTime `json:"date"`
 				} `json:"messages"`
 			}
-			if err := httpGetGerritJSON(urlStr, &obj); err != nil {
+			if err := httpGetGerritJSON(urlStr, nil, &obj); err != nil {
 				return time.Time{}, User{}, err
 			}
 			retUpdatedAt := obj.Updated.Val
@@ -316,7 +316,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 				HTMLURL string `json:"html_url"`
 			} `json:"merged_by"`
 		}
-		if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum, &obj); err != nil {
+		if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum, nil, &obj); err != nil {
 			return time.Time{}, User{}, err
 		}
 
@@ -340,7 +340,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 					HTMLURL string `json:"html_url"`
 				} `json:"user"`
 			}
-			if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", &comments, githubPagination); err != nil {
+			if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", nil, &comments, githubPagination); err != nil {
 				return time.Time{}, User{}, err
 			}
 			for _, comment := range comments {
@@ -360,7 +360,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 					HTMLURL string `json:"html_url"`
 				} `json:"user"`
 			}
-			if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", &reviewComments, githubPagination); err != nil {
+			if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", nil, &reviewComments, githubPagination); err != nil {
 				return time.Time{}, User{}, err
 			}
 			for _, comment := range reviewComments {
@@ -379,7 +379,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 					HTMLURL string `json:"html_url"`
 				} `json:"actor"`
 			}
-			if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/events", &events); err != nil {
+			if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/events", nil, &events); err != nil {
 				return time.Time{}, User{}, err
 			}
 			for _, event := range events {
@@ -403,7 +403,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 		var obj struct {
 			UpdatedAt time.Time `json:"updated_at"`
 		}
-		if err := httpGetJSON(urlStr, &obj); err != nil {
+		if err := httpGetJSON(urlStr, nil, &obj); err != nil {
 			return time.Time{}, User{}, err
 		}
 		return obj.UpdatedAt, User{}, nil
@@ -428,7 +428,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 						} `json:"committer"`
 					} `json:"commit"`
 				}
-				if err := httpGetJSON(urlStr, &obj); err != nil {
+				if err := httpGetJSON(urlStr, nil, &obj); err != nil {
 					return time.Time{}, User{}, err
 				}
 				if obj.Commit.Author.Date.After(ret) {
diff --git a/cmd/generate/src_mastodon.go b/cmd/generate/src_mastodon.go
index b4b54a8..52dcfa4 100644
--- a/cmd/generate/src_mastodon.go
+++ b/cmd/generate/src_mastodon.go
@@ -19,12 +19,12 @@ func ReadStandups(server, username string) ([]*MastodonStatus, error) {
 	var account struct {
 		ID string `json:"id"`
 	}
-	if err := httpGetJSON(server+"/api/v1/accounts/lookup?acct="+username, &account); err != nil {
+	if err := httpGetJSON(server+"/api/v1/accounts/lookup?acct="+username, nil, &account); err != nil {
 		return nil, err
 	}
 
 	var statuses []*MastodonStatus
-	if err := httpGetPaginatedJSON(server+"/api/v1/accounts/"+account.ID+"/statuses", &statuses, func(_ int) url.Values {
+	if err := httpGetPaginatedJSON(server+"/api/v1/accounts/"+account.ID+"/statuses", nil, &statuses, func(_ int) url.Values {
 		params := make(url.Values)
 		params.Set("tagged", "DailyStandUp")
 		params.Set("exclude_reblogs", "true")