feat: Iterator support via Scan and Scan2

example/iterpagination/main.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/google/go-github/v81/github"
+)
+
+func main() {
+	flag.Parse()
+	token := os.Getenv("GITHUB_AUTH_TOKEN")
+	if token == "" {
+		log.Fatal("Unauthorized: No token present")
+	}
+
+	ctx := context.Background()
+	client := github.NewClient(nil).WithAuthToken(token)
+
+	owner := "google"
+	repo := "go-github"
+	issue := 2618
+
+	opts := github.IssueListCommentsOptions{
+		Sort: github.Ptr("created"),
+		ListOptions: github.ListOptions{
+			Page:    1,
+			PerPage: 5,
+		},
+	}
+
+	fmt.Println("Listing comments for issue", issue, "in repository", owner+"/"+repo)
+
+	var paginatedCommentsCount int
+	paginatedOpts := opts
+	for {
+		pageComments, resp, err := client.Issues.ListComments(ctx, owner, repo, issue, &paginatedOpts)
+		if err != nil {
+			log.Fatalf("ListComments failed: %v", err)
+		}
+		fmt.Printf("Response: %#+v\n", resp)
+		for _, c := range pageComments {
+			body := c.GetBody()
+			if len(body) > 50 {
+				body = body[:50]
+			}
+			fmt.Printf("Comment: %q\n", body)
+		}
+		paginatedCommentsCount += len(pageComments)
+		if resp.NextPage == 0 {
+			break
+		}
+		paginatedOpts.Page = resp.NextPage
+	}
+	fmt.Println("Paginated comments:", paginatedCommentsCount)
+
+	var scannedCommentsCount int
+	scannedOpts := opts
+	for c := range github.MustIter(github.Scan2(func(p github.PaginationOption) ([]*github.IssueComment, *github.Response, error) {
+		return client.Issues.ListComments(ctx, owner, repo, issue, &scannedOpts, p)
+	})) {
+		body := c.GetBody()
+		if len(body) > 50 {
+			body = body[:50]
+		}
+		fmt.Printf("Comment: %q\n", body)
+		scannedCommentsCount++
+	}
+	fmt.Println("Scanned comments:", scannedCommentsCount)
+
+	if paginatedCommentsCount != scannedCommentsCount {
+		log.Fatalf("Mismatch in comment counts: paginated=%d scanned=%d", paginatedCommentsCount, scannedCommentsCount)
+	}
+}

github/github.go

@@ -540,6 +540,24 @@ func WithVersion(version string) RequestOption {
 	}
 }
 
+// WithOffsetPagination adds page query parameter to the request.
+func WithOffsetPagination(page int) RequestOption {
+	return func(req *http.Request) {
+		q := req.URL.Query()
+		q.Set("page", strconv.Itoa(page))
+		req.URL.RawQuery = q.Encode()
+	}
+}
+
+// WithCursorPagination adds cursor pagination parameters to the request.
+func WithCursorPagination(cursor string) RequestOption {
+	return func(req *http.Request) {
+		q := req.URL.Query()
+		// TODO
+		req.URL.RawQuery = q.Encode()
+	}
+}
+
 // NewRequest creates an API request. A relative URL can be provided in urlStr,
 // in which case it is resolved relative to the BaseURL of the Client.
 // Relative URLs should always be specified without a preceding slash. If
@@ -581,7 +599,9 @@ func (c *Client) NewRequest(method, urlStr string, body any, opts ...RequestOpti
 	req.Header.Set(headerAPIVersion, defaultAPIVersion)
 
 	for _, opt := range opts {
-		opt(req)
+		if opt != nil {
+			opt(req)
+		}
 	}
 
 	return req, nil

github/issues_comments.go

@@ -60,7 +60,7 @@ type IssueListCommentsOptions struct {
 //
 //meta:operation GET /repos/{owner}/{repo}/issues/comments
 //meta:operation GET /repos/{owner}/{repo}/issues/{issue_number}/comments
-func (s *IssuesService) ListComments(ctx context.Context, owner, repo string, number int, opts *IssueListCommentsOptions) ([]*IssueComment, *Response, error) {
+func (s *IssuesService) ListComments(ctx context.Context, owner, repo string, number int, opts *IssueListCommentsOptions, reqOpts ...RequestOption) ([]*IssueComment, *Response, error) {
 	var u string
 	if number == 0 {
 		u = fmt.Sprintf("repos/%v/%v/issues/comments", owner, repo)
@@ -72,7 +72,7 @@ func (s *IssuesService) ListComments(ctx context.Context, owner, repo string, nu
 		return nil, nil, err
 	}
 
-	req, err := s.client.NewRequest("GET", u, nil)
+	req, err := s.client.NewRequest("GET", u, nil, reqOpts...)
 	if err != nil {
 		return nil, nil, err
 	}

github/pagination.go

@@ -0,0 +1,100 @@
+package github
+
+import (
+	"fmt"
+	"iter"
+	"slices"
+)
+
+type PaginationOption = RequestOption
+
+// Scan scans all pages for the given request function f and returns individual items in an iterator.
+// If an error happens during pagination, the iterator stops immediately.
+// The caller must consume the returned error function to retrieve potential errors.
+func Scan[T any](f func(p PaginationOption) ([]T, *Response, error)) (iter.Seq[T], func() error) {
+	exhausted := false
+	var e error
+	it := func(yield func(T) bool) {
+		defer func() {
+			exhausted = true
+		}()
+		for t, err := range Scan2(f) {
+			if err != nil {
+				e = err
+				return
+			}
+
+			if !yield(t) {
+				return
+			}
+		}
+	}
+	hasErr := func() error {
+		if !exhausted {
+			panic("called error function of Scan iterator before iterator was exhausted")
+		}
+		return e
+	}
+	return it, hasErr
+}
+
+// Scan2 scans all pages for the given request function f and returns individual items and potential errors in an iterator.
+// The caller must consume the error element of the iterator during each iteration
+// to ensure that no errors happened.
+func Scan2[T any](f func(p PaginationOption) ([]T, *Response, error)) iter.Seq2[T, error] {
+	return func(yield func(T, error) bool) {
+		var nextOpt PaginationOption
+
+	Pagination:
+		for {
+			ts, resp, err := f(nextOpt)
+			if err != nil {
+				var t T
+				yield(t, err)
+				return
+			}
+
+			for _, t := range ts {
+				if !yield(t, nil) {
+					return
+				}
+			}
+
+			// the f request function was either configured for offset- or cursor-based pagination.
+			switch {
+			case resp.NextPage != 0:
+				nextOpt = WithOffsetPagination(resp.NextPage)
+			case resp.Cursor != "":
+				nextOpt = WithCursorPagination(resp.Cursor)
+			default:
+				// no more pages
+				break Pagination
+			}
+		}
+	}
+}
+
+// MustIter provides a single item iterator for the provided two item iterator and panics if an error happens.
+func MustIter[T any](it iter.Seq2[T, error]) iter.Seq[T] {
+	return func(yield func(T) bool) {
+		for x, err := range it {
+			if err != nil {
+				panic(fmt.Errorf("iterator produced an error: %w", err))
+			}
+
+			if !yield(x) {
+				return
+			}
+		}
+	}
+}
+
+// ScanAndCollect is a convenience function that collects all results and returns them as slice as well as an error if one happens.
+func ScanAndCollect[T any](f func(p PaginationOption) ([]T, *Response, error)) ([]T, error) {
+	it, hasErr := Scan(f)
+	allItems := slices.Collect(it)
+	if err := hasErr(); err != nil {
+		return nil, err
+	}
+	return allItems, nil
+}

github/pagination_test.go

@@ -0,0 +1,153 @@
+package github
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+// FixtureManager handles loading and updating test fixtures.
+type FixtureManager struct {
+	dir        string
+	updateMode bool
+	token      string
+	client     *Client
+}
+
+// NewFixtureManager creates a new fixture manager.
+func NewFixtureManager(t *testing.T, dir string) *FixtureManager {
+	updateMode := os.Getenv("GITHUB_UPDATE_FIXTURE") != ""
+	token := os.Getenv("GITHUB_AUTH_TOKEN")
+
+	var client *Client
+	if updateMode && token != "" {
+		client = NewClient(nil).WithAuthToken(token)
+	}
+
+	return &FixtureManager{
+		dir:        dir,
+		updateMode: updateMode,
+		token:      token,
+		client:     client,
+	}
+}
+
+// LoadOrFetch loads a fixture from file or fetches from GitHub API and saves it.
+// The fetch function receives the client and should make a real API call.
+func (fm *FixtureManager) LoadOrFetch(t *testing.T, name string, fetch func(*Client) (any, error)) []byte {
+	path := filepath.Join(fm.dir, name+".json")
+
+	if fm.updateMode {
+		t.Logf("Updating fixture: %s from GitHub API", name)
+		data, err := fetch(fm.client)
+		if err != nil {
+			t.Fatalf("Failed to fetch fixture data from GitHub API: %v", err)
+		}
+
+		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			t.Fatalf("Failed to create fixture directory: %v", err)
+		}
+
+		// Marshal to JSON
+		prettyBytes, err := json.MarshalIndent(data, "", "  ")
+		if err != nil {
+			t.Fatalf("Failed to marshal fixture data: %v", err)
+		}
+
+		if err := os.WriteFile(path, prettyBytes, 0644); err != nil {
+			t.Fatalf("Failed to write fixture file: %v", err)
+		}
+
+		return prettyBytes
+	}
+
+	bytes, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("Failed to load fixture file %s: %v", path, err)
+	}
+
+	return bytes
+}
+
+func TestPagination_Scan2_ListComments(t *testing.T) {
+	t.Parallel()
+
+	client, mux, _ := setup(t)
+	fm := NewFixtureManager(t, filepath.Join("testdata", "pagination"))
+
+	issue := 2618
+	mux.HandleFunc(fmt.Sprintf("/repos/google/go-github/issues/%d/comments", issue), func(w http.ResponseWriter, r *http.Request) {
+		testMethod(t, r, "GET")
+		page := r.URL.Query().Get("page")
+		var fixture []byte
+		loaderFn := func(page int) func(client *Client) (any, error) {
+			opts := IssueListCommentsOptions{
+				ListOptions: ListOptions{
+					PerPage: 5,
+					Page:    page,
+				},
+			}
+			return func(client *Client) (any, error) {
+				comments, resp, err := client.Issues.ListComments(t.Context(), "google", "go-github", issue, &opts)
+				fmt.Printf("Link: %s\n\n", resp.Header["Link"][0])
+				return comments, err
+			}
+		}
+		switch page {
+		case "", "1":
+			w.Header().Set("Link", `<https://api.github.com/repositories/10270722/issues/2618/comments?page=2&per_page=5>; rel="next", <https://api.github.com/repositories/10270722/issues/2618/comments?page=5&per_page=5>; rel="last"`)
+			fixture = fm.LoadOrFetch(t, "list_comments_page1", loaderFn(1))
+			w.Write(fixture)
+		case "2":
+			w.Header().Set("Link", `<https://api.github.com/repositories/10270722/issues/2618/comments?page=1&per_page=5>; rel="prev", <https://api.github.com/repositories/10270722/issues/2618/comments?page=3&per_page=5>; rel="next", <https://api.github.com/repositories/10270722/issues/2618/comments?page=5&per_page=5>; rel="last", <https://api.github.com/repositories/10270722/issues/2618/comments?page=1&per_page=5>; rel="first"`)
+			fixture = fm.LoadOrFetch(t, "list_comments_page2", loaderFn(2))
+			w.Write(fixture)
+		case "3":
+			w.Header().Set("Link", `<https://api.github.com/repositories/10270722/issues/2618/comments?page=2&per_page=5>; rel="prev", <https://api.github.com/repositories/10270722/issues/2618/comments?page=4&per_page=5>; rel="next", <https://api.github.com/repositories/10270722/issues/2618/comments?page=5&per_page=5>; rel="last", <https://api.github.com/repositories/10270722/issues/2618/comments?page=1&per_page=5>; rel="first"`)
+			fixture = fm.LoadOrFetch(t, "list_comments_page3", loaderFn(3))
+			w.Write(fixture)
+		case "4":
+			w.Header().Set("Link", `<https://api.github.com/repositories/10270722/issues/2618/comments?page=3&per_page=5>; rel="prev", <https://api.github.com/repositories/10270722/issues/2618/comments?page=5&per_page=5>; rel="next", <https://api.github.com/repositories/10270722/issues/2618/comments?page=5&per_page=5>; rel="last", <https://api.github.com/repositories/10270722/issues/2618/comments?page=1&per_page=5>; rel="first"`)
+			fixture = fm.LoadOrFetch(t, "list_comments_page4", loaderFn(4))
+			w.Write(fixture)
+		case "5":
+			w.Header().Set("Link", `<https://api.github.com/repositories/10270722/issues/2618/comments?page=4&per_page=5>; rel="prev", <https://api.github.com/repositories/10270722/issues/2618/comments?page=1&per_page=5>; rel="first"`)
+			fixture = fm.LoadOrFetch(t, "list_comments_page5", loaderFn(5))
+			w.Write(fixture)
+		}
+	})
+
+	ctx := t.Context()
+	opts := &IssueListCommentsOptions{}
+
+	var comments []*IssueComment
+	for c, err := range Scan2(func(p PaginationOption) ([]*IssueComment, *Response, error) {
+		return client.Issues.ListComments(ctx, "google", "go-github", issue, opts, p)
+	}) {
+		if err != nil {
+			t.Fatalf("Scan2 iterator returned error: %v", err)
+		}
+		comments = append(comments, c)
+	}
+
+	wantCommentIDs := []int64{
+		1372144817, 1386971275, 1396478534, 1446756890, 1656588501,
+		1659541626, 1664125370, 1684770158, 1685120763, 1884764520,
+		1912173472, 1912236179, 1912244287, 1912386258, 1919918050,
+		1919936396, 1919948684, 1920002236, 1920009324, 1920084186,
+		1979228975, 1994323766, 1994383750, 2708656405,
+	}
+	commentIDs := make([]int64, len(comments))
+	for i, c := range comments {
+		commentIDs[i] = c.GetID()
+	}
+
+	if !cmp.Equal(commentIDs, wantCommentIDs) {
+		t.Errorf("Got %+v, want %+v", commentIDs, wantCommentIDs)
+	}
+}