registry: Add ListManfiests and ListRepositoriesV2 api endpoint support + Token pagination (#501)

* registry: Added ListRegistryManifests and ListRepositoriesV2 * Updated README with toke pagination example * Updated README with toke pagination example * Renamed links helper to be more descriptive * Added link page token tests * registry: Removed unnecessary links/meta assignment * registry: Fix links/meta assignment Co-authored-by: Collin Shoop <cshoop@digitalocean.com>
2021-12-03 10:07:08 -05:00 · 2021-12-03 10:07:08 -05:00 · 1f3cbb437c
parent f3da1960d4
commit 1f3cbb437c
7 changed files with 467 additions and 13 deletions
--- a/README.md
+++ b/README.md
@ -118,6 +118,43 @@ func DropletList(ctx context.Context, client *godo.Client) ([]godo.Droplet, erro
 }
 ```

+Some endpoints offer token based pagination. For example, to fetch all Registry Repositories:
+
+```go
+func ListRepositoriesV2(ctx context.Context, client *godo.Client, registryName string) ([]*godo.RepositoryV2, error) {
+    // create a list to hold our registries
+    list := []*godo.RepositoryV2{}
+
+    // create options. initially, these will be blank
+    opt := &godo.TokenListOptions{}
+    for {
+        repositories, resp, err := client.Registry.ListRepositoriesV2(ctx, registryName, opt)
+        if err != nil {
+            return nil, err
+        }
+
+        // append the current page's registries to our list
+        list = append(list, repositories...)
+
+        // if we are at the last page, break out the for loop
+        if resp.Links == nil || resp.Links.IsLastPage() {
+            break
+        }
+
+        // grab the next page token
+        nextPageToken, err := resp.Links.NextPageToken()
+        if err != nil {
+            return nil, err
+        }
+
+        // provide the next page token for the next request
+        opt.Token = nextPageToken
+    }
+
+    return list, nil
+}
+```
+
 ## Versioning

 Each version of the client is tagged and the version is updated accordingly.
--- a/godo.go
+++ b/godo.go
@ -99,6 +99,20 @@ type ListOptions struct {
 	PerPage int `url:"per_page,omitempty"`
 }

+// TokenListOptions specifies the optional parameters to various List methods that support token pagination.
+type TokenListOptions struct {
+	// For paginated result sets, page of results to retrieve.
+	Page int `url:"page,omitempty"`
+
+	// For paginated result sets, the number of results to include per page.
+	PerPage int `url:"per_page,omitempty"`
+
+	// For paginated result sets which support tokens, the token provided by the last set
+	// of results in order to retrieve the next set of results. This is expected to be faster
+	// than incrementing or decrementing the page number.
+	Token string `url:"page_token,omitempty"`
+}
+
 // Response is a DigitalOcean response. This wraps the standard http.Response returned from DigitalOcean.
 type Response struct {
 	*http.Response
--- a/godo_test.go
+++ b/godo_test.go
@ -611,6 +611,32 @@ func checkCurrentPage(t *testing.T, resp *Response, expectedPage int) {
 	}
 }

+func checkNextPageToken(t *testing.T, resp *Response, expectedNextPageToken string) {
+	t.Helper()
+	links := resp.Links
+	pageToken, err := links.NextPageToken()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if pageToken != expectedNextPageToken {
+		t.Fatalf("expected next page token to be '%s', was '%s'", expectedNextPageToken, pageToken)
+	}
+}
+
+func checkPreviousPageToken(t *testing.T, resp *Response, expectedPreviousPageToken string) {
+	t.Helper()
+	links := resp.Links
+	pageToken, err := links.PrevPageToken()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if pageToken != expectedPreviousPageToken {
+		t.Fatalf("expected previous page token to be '%s', was '%s'", expectedPreviousPageToken, pageToken)
+	}
+}
+
 func TestDo_completion_callback(t *testing.T) {
 	setup()
 	defer teardown()
--- a/links.go
+++ b/links.go
@ -32,6 +32,16 @@ func (l *Links) CurrentPage() (int, error) {
 	return l.Pages.current()
 }

+// NextPageToken is the page token to request the next page of the list
+func (l *Links) NextPageToken() (string, error) {
+	return l.Pages.nextPageToken()
+}
+
+// PrevPageToken is the page token to request the previous page of the list
+func (l *Links) PrevPageToken() (string, error) {
+	return l.Pages.prevPageToken()
+}
+
 func (p *Pages) current() (int, error) {
 	switch {
 	case p == nil:
@ -50,6 +60,28 @@ func (p *Pages) current() (int, error) {
 	return 0, nil
 }

+func (p *Pages) nextPageToken() (string, error) {
+	if p == nil || p.Next == "" {
+		return "", nil
+	}
+	token, err := pageTokenFromURL(p.Next)
+	if err != nil {
+		return "", err
+	}
+	return token, nil
+}
+
+func (p *Pages) prevPageToken() (string, error) {
+	if p == nil || p.Prev == "" {
+		return "", nil
+	}
+	token, err := pageTokenFromURL(p.Prev)
+	if err != nil {
+		return "", err
+	}
+	return token, nil
+}
+
 // IsLastPage returns true if the current page is the last
 func (l *Links) IsLastPage() bool {
 	if l.Pages == nil {
@ -77,6 +109,14 @@ func pageForURL(urlText string) (int, error) {
 	return page, nil
 }

+func pageTokenFromURL(urlText string) (string, error) {
+	u, err := url.ParseRequestURI(urlText)
+	if err != nil {
+		return "", err
+	}
+	return u.Query().Get("page_token"), nil
+}
+
 // Get a link action by id.
 func (la *LinkAction) Get(ctx context.Context, client *Client) (*Action, *Response, error) {
 	return client.Actions.Get(ctx, la.ID)
--- a/links_test.go
+++ b/links_test.go
@ -130,25 +130,32 @@ func TestLinks_ParseMissing(t *testing.T) {

 func TestLinks_ParseURL(t *testing.T) {
 	type linkTest struct {
-		name, url string
-		expected  int
+		name, url         string
+		expectedPage      int
+		expectedPageToken string
 	}

 	linkTests := []linkTest{
 		{
-			name:     "prev",
-			url:      "https://api.digitalocean.com/v2/droplets/?page=1",
-			expected: 1,
+			name:         "prev",
+			url:          "https://api.digitalocean.com/v2/droplets/?page=1",
+			expectedPage: 1,
 		},
 		{
-			name:     "last",
-			url:      "https://api.digitalocean.com/v2/droplets/?page=5",
-			expected: 5,
+			name:         "last",
+			url:          "https://api.digitalocean.com/v2/droplets/?page=5",
+			expectedPage: 5,
 		},
 		{
-			name:     "nexta",
-			url:      "https://api.digitalocean.com/v2/droplets/?page=2",
-			expected: 2,
+			name:         "next",
+			url:          "https://api.digitalocean.com/v2/droplets/?page=2",
+			expectedPage: 2,
+		},
+		{
+			name:              "page token",
+			url:               "https://api.digitalocean.com/v2/droplets/?page=2&page_token=aaa",
+			expectedPage:      2,
+			expectedPageToken: "aaa",
 		},
 	}

@ -158,9 +165,15 @@ func TestLinks_ParseURL(t *testing.T) {
 			t.Fatal(err)
 		}

-		if p != lT.expected {
+		if p != lT.expectedPage {
 			t.Errorf("expected page for '%s' to be '%d', was '%d'",
-				lT.url, lT.expected, p)
+				lT.url, lT.expectedPage, p)
+		}
+
+		pageToken, err := pageTokenFromURL(lT.url)
+		if pageToken != lT.expectedPageToken {
+			t.Errorf("expected pageToken for '%s' to be '%s', was '%s'",
+				lT.url, lT.expectedPageToken, pageToken)
 		}
 	}

@ -197,3 +210,47 @@ func TestLinks_ParseEmptyString(t *testing.T) {
 		}
 	}
 }
+
+func TestLinks_NextPageToken(t *testing.T) {
+	t.Run("happy token", func(t *testing.T) {
+		checkNextPageToken(t, &Response{Links: &Links{
+			Pages: &Pages{
+				Next: "https://api.digitalocean.com/v2/droplets/?page_token=aaa",
+			},
+		}}, "aaa")
+	})
+	t.Run("empty token", func(t *testing.T) {
+		checkNextPageToken(t, &Response{Links: &Links{
+			Pages: &Pages{
+				Next: "https://api.digitalocean.com/v2/droplets/",
+			},
+		}}, "")
+	})
+	t.Run("no next page", func(t *testing.T) {
+		checkNextPageToken(t, &Response{Links: &Links{
+			Pages: &Pages{},
+		}}, "")
+	})
+}
+
+func TestLinks_ParseNextPageToken(t *testing.T) {
+	t.Run("happy token", func(t *testing.T) {
+		checkPreviousPageToken(t, &Response{Links: &Links{
+			Pages: &Pages{
+				Prev: "https://api.digitalocean.com/v2/droplets/?page_token=aaa",
+			},
+		}}, "aaa")
+	})
+	t.Run("empty token", func(t *testing.T) {
+		checkPreviousPageToken(t, &Response{Links: &Links{
+			Pages: &Pages{
+				Prev: "https://api.digitalocean.com/v2/droplets/",
+			},
+		}}, "")
+	})
+	t.Run("no next page", func(t *testing.T) {
+		checkPreviousPageToken(t, &Response{Links: &Links{
+			Pages: &Pages{},
+		}}, "")
+	})
+}
--- a/registry.go
+++ b/registry.go
@ -25,8 +25,10 @@ type RegistryService interface {
 	Delete(context.Context) (*Response, error)
 	DockerCredentials(context.Context, *RegistryDockerCredentialsRequest) (*DockerCredentials, *Response, error)
 	ListRepositories(context.Context, string, *ListOptions) ([]*Repository, *Response, error)
+	ListRepositoriesV2(context.Context, string, *TokenListOptions) ([]*RepositoryV2, *Response, error)
 	ListRepositoryTags(context.Context, string, string, *ListOptions) ([]*RepositoryTag, *Response, error)
 	DeleteTag(context.Context, string, string, string) (*Response, error)
+	ListRepositoryManifests(context.Context, string, string, *ListOptions) ([]*RepositoryManifest, *Response, error)
 	DeleteManifest(context.Context, string, string, string) (*Response, error)
 	StartGarbageCollection(context.Context, string, ...*StartGarbageCollectionRequest) (*GarbageCollection, *Response, error)
 	GetGarbageCollection(context.Context, string) (*GarbageCollection, *Response, error)
@ -73,6 +75,15 @@ type Repository struct {
 	TagCount     uint64         `json:"tag_count,omitempty"`
 }

+// RepositoryV2 represents a repository in the V2 format
+type RepositoryV2 struct {
+	RegistryName   string              `json:"registry_name,omitempty"`
+	Name           string              `json:"name,omitempty"`
+	TagCount       uint64              `json:"tag_count,omitempty"`
+	ManifestCount  uint64              `json:"manifest_count,omitempty"`
+	LatestManifest *RepositoryManifest `json:"latest_manifest,omitempty"`
+}
+
 // RepositoryTag represents a repository tag
 type RepositoryTag struct {
 	RegistryName        string    `json:"registry_name,omitempty"`
@ -84,6 +95,24 @@ type RepositoryTag struct {
 	UpdatedAt           time.Time `json:"updated_at,omitempty"`
 }

+// RepositoryManifest represents a repository manifest
+type RepositoryManifest struct {
+	RegistryName        string    `json:"registry_name,omitempty"`
+	Repository          string    `json:"repository,omitempty"`
+	Digest              string    `json:"digest,omitempty"`
+	CompressedSizeBytes uint64    `json:"compressed_size_bytes,omitempty"`
+	SizeBytes           uint64    `json:"size_bytes,omitempty"`
+	UpdatedAt           time.Time `json:"updated_at,omitempty"`
+	Tags                []string  `json:"tags,omitempty"`
+	Blobs               []*Blob   `json:"blobs,omitempty"`
+}
+
+// Blob represents a registry blob
+type Blob struct {
+	Digest              string `json:"digest,omitempty"`
+	CompressedSizeBytes uint64 `json:"compressed_size_bytes,omitempty"`
+}
+
 type registryRoot struct {
 	Registry *Registry `json:"registry,omitempty"`
 }
@ -94,12 +123,24 @@ type repositoriesRoot struct {
 	Meta         *Meta         `json:"meta"`
 }

+type repositoriesV2Root struct {
+	Repositories []*RepositoryV2 `json:"repositories,omitempty"`
+	Links        *Links          `json:"links,omitempty"`
+	Meta         *Meta           `json:"meta"`
+}
+
 type repositoryTagsRoot struct {
 	Tags  []*RepositoryTag `json:"tags,omitempty"`
 	Links *Links           `json:"links,omitempty"`
 	Meta  *Meta            `json:"meta"`
 }

+type repositoryManifestsRoot struct {
+	Manifests []*RepositoryManifest `json:"manifests,omitempty"`
+	Links     *Links                `json:"links,omitempty"`
+	Meta      *Meta                 `json:"meta"`
+}
+
 // GarbageCollection represents a garbage collection.
 type GarbageCollection struct {
 	UUID         string                `json:"uuid"`
@ -293,6 +334,30 @@ func (svc *RegistryServiceOp) ListRepositories(ctx context.Context, registry str
 	return root.Repositories, resp, nil
 }

+// ListRepositoriesV2 returns a list of the Repositories in a registry.
+func (svc *RegistryServiceOp) ListRepositoriesV2(ctx context.Context, registry string, opts *TokenListOptions) ([]*RepositoryV2, *Response, error) {
+	path := fmt.Sprintf("%s/%s/repositoriesV2", registryPath, registry)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(repositoriesV2Root)
+
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	resp.Links = root.Links
+	resp.Meta = root.Meta
+
+	return root.Repositories, resp, nil
+}
+
 // ListRepositoryTags returns a list of the RepositoryTags available within the given repository.
 func (svc *RegistryServiceOp) ListRepositoryTags(ctx context.Context, registry, repository string, opts *ListOptions) ([]*RepositoryTag, *Response, error) {
 	path := fmt.Sprintf("%s/%s/repositories/%s/tags", registryPath, registry, url.PathEscape(repository))
@ -336,6 +401,30 @@ func (svc *RegistryServiceOp) DeleteTag(ctx context.Context, registry, repositor
 	return resp, nil
 }

+// ListRepositoryManifests returns a list of the RepositoryManifests available within the given repository.
+func (svc *RegistryServiceOp) ListRepositoryManifests(ctx context.Context, registry, repository string, opts *ListOptions) ([]*RepositoryManifest, *Response, error) {
+	path := fmt.Sprintf("%s/%s/repositories/%s/digests", registryPath, registry, url.PathEscape(repository))
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(repositoryManifestsRoot)
+
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	resp.Links = root.Links
+	resp.Meta = root.Meta
+
+	return root.Manifests, resp, nil
+}
+
 // DeleteManifest deletes a manifest by its digest within a given repository.
 func (svc *RegistryServiceOp) DeleteManifest(ctx context.Context, registry, repository, digest string) (*Response, error) {
 	path := fmt.Sprintf("%s/%s/repositories/%s/digests/%s", registryPath, registry, url.PathEscape(repository), digest)
--- a/registry_test.go
+++ b/registry_test.go
@ -265,6 +265,109 @@ func TestRepository_List(t *testing.T) {
 	assert.Equal(t, wantRespMeta, gotRespMeta)
 }

+func TestRepository_ListV2(t *testing.T) {
+	setup()
+	defer teardown()
+
+	wantRepositories := []*RepositoryV2{
+		{
+			RegistryName:  testRegistry,
+			Name:          testRepository,
+			TagCount:      2,
+			ManifestCount: 1,
+			LatestManifest: &RepositoryManifest{
+				Digest:              "sha256:abc",
+				RegistryName:        testRegistry,
+				Repository:          testRepository,
+				CompressedSizeBytes: testCompressedSize,
+				SizeBytes:           testSize,
+				UpdatedAt:           testTime,
+				Tags:                []string{"v1", "v2"},
+				Blobs: []*Blob{
+					{
+						Digest:              "sha256:blob1",
+						CompressedSizeBytes: 100,
+					},
+					{
+						Digest:              "sha256:blob2",
+						CompressedSizeBytes: 200,
+					},
+				},
+			},
+		},
+	}
+	baseLinkPage := fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositoriesV2", testRegistry)
+	getResponseJSON := `{
+	"repositories": [
+		{
+			"registry_name": "` + testRegistry + `",
+			"name": "` + testRepository + `",
+			"tag_count": 2,
+			"manifest_count": 1,
+			"latest_manifest": {
+				"digest": "sha256:abc",
+				"registry_name": "` + testRegistry + `",
+				"repository": "` + testRepository + `",
+				"compressed_size_bytes": ` + fmt.Sprintf("%d", testCompressedSize) + `,
+				"size_bytes": ` + fmt.Sprintf("%d", testSize) + `,
+				"updated_at": "` + testTimeString + `",
+				"tags": [
+					"v1",
+					"v2"
+				],
+				"blobs": [
+					{
+						"digest": "sha256:blob1",
+						"compressed_size_bytes": 100
+					},
+					{
+						"digest": "sha256:blob2",
+						"compressed_size_bytes": 200
+					}
+				]
+			}
+		}
+	],
+	"links": {
+	    "pages": {
+			"first":    "` + baseLinkPage + `?page=1&page_size=1",
+			"prev":     "` + baseLinkPage + `?page=2&page_size=1&page_token=aaa",
+			"next":     "` + baseLinkPage + `?page=4&page_size=1&page_token=ccc",
+			"last":     "` + baseLinkPage + `?page=5&page_size=1"
+		}
+	},
+	"meta": {
+	    "total": 5
+	}
+}`
+
+	mux.HandleFunc(fmt.Sprintf("/v2/registry/%s/repositoriesV2", testRegistry), func(w http.ResponseWriter, r *http.Request) {
+		testMethod(t, r, http.MethodGet)
+		testFormValues(t, r, map[string]string{"page": "3", "per_page": "1", "page_token": "bbb"})
+		fmt.Fprint(w, getResponseJSON)
+	})
+	got, response, err := client.Registry.ListRepositoriesV2(ctx, testRegistry, &TokenListOptions{Page: 3, PerPage: 1, Token: "bbb"})
+	require.NoError(t, err)
+	require.Equal(t, wantRepositories, got)
+
+	gotRespLinks := response.Links
+	wantRespLinks := &Links{
+		Pages: &Pages{
+			First: fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositoriesV2?page=1&page_size=1", testRegistry),
+			Prev:  fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositoriesV2?page=2&page_size=1&page_token=aaa", testRegistry),
+			Next:  fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositoriesV2?page=4&page_size=1&page_token=ccc", testRegistry),
+			Last:  fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositoriesV2?page=5&page_size=1", testRegistry),
+		},
+	}
+	assert.Equal(t, wantRespLinks, gotRespLinks)
+
+	gotRespMeta := response.Meta
+	wantRespMeta := &Meta{
+		Total: 5,
+	}
+	assert.Equal(t, wantRespMeta, gotRespMeta)
+}
+
 func TestRepository_ListTags(t *testing.T) {
 	setup()
 	defer teardown()
@ -340,6 +443,94 @@ func TestRegistry_DeleteTag(t *testing.T) {
 	require.NoError(t, err)
 }

+func TestRegistry_ListManifests(t *testing.T) {
+	setup()
+	defer teardown()
+
+	wantTags := []*RepositoryManifest{
+		{
+			RegistryName:        testRegistry,
+			Repository:          testRepository,
+			Digest:              testDigest,
+			CompressedSizeBytes: testCompressedSize,
+			SizeBytes:           testSize,
+			UpdatedAt:           testTime,
+			Tags:                []string{"latest", "v1", "v2"},
+			Blobs: []*Blob{
+				{
+					Digest:              "sha256:blob1",
+					CompressedSizeBytes: 998,
+				},
+				{
+					Digest:              "sha256:blob2",
+					CompressedSizeBytes: 1,
+				},
+			},
+		},
+	}
+	getResponseJSON := `{
+	"manifests": [
+		{
+			"registry_name": "` + testRegistry + `",
+			"repository": "` + testRepository + `",
+			"digest": "` + testDigest + `",
+			"compressed_size_bytes": ` + fmt.Sprintf("%d", testCompressedSize) + `,
+			"size_bytes": ` + fmt.Sprintf("%d", testSize) + `,
+			"updated_at": "` + testTimeString + `",
+			"tags": [ "latest", "v1", "v2" ],
+			"blobs": [
+				{
+					"digest": "sha256:blob1",
+					"compressed_size_bytes": 998
+				}, 
+				{
+
+					"digest": "sha256:blob2",
+					"compressed_size_bytes": 1
+				}
+			]
+		}
+	],
+	"links": {
+	    "pages": {
+			"first": "https://api.digitalocean.com/v2/registry/` + testRegistry + `/repositories/` + testEncodedRepository + `/digests?page=1&page_size=1",
+			"prev": "https://api.digitalocean.com/v2/registry/` + testRegistry + `/repositories/` + testEncodedRepository + `/digests?page=2&page_size=1",
+			"next": "https://api.digitalocean.com/v2/registry/` + testRegistry + `/repositories/` + testEncodedRepository + `/digests?page=4&page_size=1",
+			"last": "https://api.digitalocean.com/v2/registry/` + testRegistry + `/repositories/` + testEncodedRepository + `/digests?page=5&page_size=1"
+		}
+	},
+	"meta": {
+	    "total": 5
+	}
+}`
+
+	mux.HandleFunc(fmt.Sprintf("/v2/registry/%s/repositories/%s/digests", testRegistry, testRepository), func(w http.ResponseWriter, r *http.Request) {
+		testMethod(t, r, http.MethodGet)
+		testFormValues(t, r, map[string]string{"page": "3", "per_page": "1"})
+		fmt.Fprint(w, getResponseJSON)
+	})
+	got, response, err := client.Registry.ListRepositoryManifests(ctx, testRegistry, testRepository, &ListOptions{Page: 3, PerPage: 1})
+	require.NoError(t, err)
+	require.Equal(t, wantTags, got)
+
+	gotRespLinks := response.Links
+	wantRespLinks := &Links{
+		Pages: &Pages{
+			First: fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositories/%s/digests?page=1&page_size=1", testRegistry, testEncodedRepository),
+			Prev:  fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositories/%s/digests?page=2&page_size=1", testRegistry, testEncodedRepository),
+			Next:  fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositories/%s/digests?page=4&page_size=1", testRegistry, testEncodedRepository),
+			Last:  fmt.Sprintf("https://api.digitalocean.com/v2/registry/%s/repositories/%s/digests?page=5&page_size=1", testRegistry, testEncodedRepository),
+		},
+	}
+	assert.Equal(t, wantRespLinks, gotRespLinks)
+
+	gotRespMeta := response.Meta
+	wantRespMeta := &Meta{
+		Total: 5,
+	}
+	assert.Equal(t, wantRespMeta, gotRespMeta)
+}
+
 func TestRegistry_DeleteManifest(t *testing.T) {
 	setup()
 	defer teardown()