From 45142ecc55014a1d2c6ed45d553b5b99ce9e55a4 Mon Sep 17 00:00:00 2001 From: tonic Date: Thu, 18 Jun 2026 19:47:30 +0800 Subject: [PATCH 1/2] feat(server): redirect blob GETs to presigned object-store URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v2GetBlob proxies every blob byte through the epoch process: it reads from the object store and io.Copy's the body back to the client, paying TLS on both sides for multi-GiB VM disk/memory blobs. A single stream tops out at ~20 MB/s even with 3 CPUs, so a 10.5 GB snapshot pull takes ~9 min, and concurrent transfers saturate the pod. When EPOCH_BLOB_REDIRECT is set, v2GetBlob now responds with a 307 to a presigned object-store URL (minio-go PresignedGetObject), so blob bytes flow client<->storage directly and epoch leaves the data path entirely. GCS honors the SigV4-signed URL and serves Range requests natively. Measured on a same-region GCE VM pulling the 5.7 GB memory-ranges blob: path single-stream 10.5 GB pull proxy (1 CPU, before) 3.4 MB/s ~51 min proxy (3 CPU) 20 MB/s ~9 min presigned direct-to-GCS 275 MB/s ~38 s ~13x over the proxy with zero client changes — any redirect-following OCI client (registryclient/vk-cocoon, oras, crane, docker) benefits transparently, and CopyBlobExact still verifies the digest end to end. Intra-blob Range parallelism (measured 1.28 GB/s at 8 connections) would need a client-side downloader and is out of scope here. The flag defaults off. On any server-side failure (existence check or presign) the handler falls back to streaming. Note there is no fallback once the 307 is sent: every client hitting /v2/blobs must have egress to the object-store host when the flag is enabled. --- objectstore/client.go | 10 +++++++ registry/registry.go | 6 ++++ server/registry_v2_blobs.go | 47 ++++++++++++++++++++++++++++++++ server/registry_v2_blobs_test.go | 25 +++++++++++++++++ server/server.go | 30 +++++++++++++------- 5 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 server/registry_v2_blobs_test.go diff --git a/objectstore/client.go b/objectstore/client.go index f789844..3db8db4 100644 --- a/objectstore/client.go +++ b/objectstore/client.go @@ -58,6 +58,16 @@ func (c *Client) Put(ctx context.Context, key string, body io.Reader, size int64 return nil } +// PresignGet returns a time-limited URL that lets a client GET the object +// directly from the backing store, bypassing this process. +func (c *Client) PresignGet(ctx context.Context, key string, ttl time.Duration) (string, error) { + u, err := c.client.PresignedGetObject(ctx, c.cfg.Bucket, c.fullKey(key), ttl, nil) + if err != nil { + return "", fmt.Errorf("presign get %s: %w", key, err) + } + return u.String(), nil +} + // Get returns a streaming reader and size for the given key. func (c *Client) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { obj, err := c.client.GetObject(ctx, c.cfg.Bucket, c.fullKey(key), minio.GetObjectOptions{}) diff --git a/registry/registry.go b/registry/registry.go index 5dc7799..4c6cb1b 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -74,6 +74,12 @@ func (r *Registry) StreamBlob(ctx context.Context, digest string) (io.ReadCloser return r.client.Get(ctx, blobKey(digest)) } +// PresignBlobGet returns a time-limited URL for fetching a blob directly from +// the object store, bypassing this process. +func (r *Registry) PresignBlobGet(ctx context.Context, digest string, ttl time.Duration) (string, error) { + return r.client.PresignGet(ctx, blobKey(digest), ttl) +} + // BlobSize returns the size of a blob in bytes. func (r *Registry) BlobSize(ctx context.Context, digest string) (int64, error) { return r.client.Head(ctx, blobKey(digest)) diff --git a/server/registry_v2_blobs.go b/server/registry_v2_blobs.go index 34463d4..7f27088 100644 --- a/server/registry_v2_blobs.go +++ b/server/registry_v2_blobs.go @@ -4,13 +4,35 @@ import ( "io" "net/http" "strconv" + "time" + + "github.com/projecteru2/core/log" "github.com/cocoonstack/epoch/manifest" ) +const defaultBlobRedirectTTL = time.Hour + +// resolveBlobRedirectTTL parses EPOCH_BLOB_REDIRECT_TTL, falling back to the +// default for empty, unparseable, or non-positive values. +func resolveBlobRedirectTTL(raw string) time.Duration { + if raw == "" { + return defaultBlobRedirectTTL + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return defaultBlobRedirectTTL + } + return d +} + func (s *Server) v2GetBlob(w http.ResponseWriter, r *http.Request) { dgst := stripSHA256Prefix(urlVar(r, "digest")) + if s.blobRedirect && s.redirectBlob(w, r, dgst) { + return + } + body, size, err := s.reg.StreamBlob(r.Context(), dgst) if err != nil { if isNotFound(err) { @@ -31,6 +53,31 @@ func (s *Server) v2GetBlob(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(w, body) } +// redirectBlob points the client at a presigned object-store URL so blob bytes +// flow directly from storage instead of being proxied through this process. +// Returns false without writing a response when the blob can't be redirected, +// so v2GetBlob falls back to streaming. +func (s *Server) redirectBlob(w http.ResponseWriter, r *http.Request, dgst string) bool { + logger := log.WithFunc("server.redirectBlob") + exists, err := s.reg.BlobExists(r.Context(), dgst) + if err != nil { + logger.Warnf(r.Context(), "blob exists check for %s failed, falling back to proxy: %v", dgst, err) + return false + } + if !exists { + v2Error(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob not found") + return true + } + url, err := s.reg.PresignBlobGet(r.Context(), dgst, s.blobRedirectTTL) + if err != nil { + logger.Warnf(r.Context(), "presign blob %s failed, falling back to proxy: %v", dgst, err) + return false + } + w.Header().Set("Docker-Content-Digest", "sha256:"+dgst) + http.Redirect(w, r, url, http.StatusTemporaryRedirect) + return true +} + func (s *Server) v2HeadBlob(w http.ResponseWriter, r *http.Request) { dgst := stripSHA256Prefix(urlVar(r, "digest")) diff --git a/server/registry_v2_blobs_test.go b/server/registry_v2_blobs_test.go new file mode 100644 index 0000000..0a31c0e --- /dev/null +++ b/server/registry_v2_blobs_test.go @@ -0,0 +1,25 @@ +package server + +import ( + "testing" + "time" +) + +func TestResolveBlobRedirectTTL(t *testing.T) { + cases := []struct { + raw string + want time.Duration + }{ + {"", defaultBlobRedirectTTL}, + {"30m", 30 * time.Minute}, + {"2h", 2 * time.Hour}, + {"garbage", defaultBlobRedirectTTL}, + {"0", defaultBlobRedirectTTL}, + {"-5m", defaultBlobRedirectTTL}, + } + for _, c := range cases { + if got := resolveBlobRedirectTTL(c.raw); got != c.want { + t.Errorf("resolveBlobRedirectTTL(%q) = %s, want %s", c.raw, got, c.want) + } + } +} diff --git a/server/server.go b/server/server.go index c734e63..50108e3 100644 --- a/server/server.go +++ b/server/server.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "os" + "strconv" "time" commonhttpx "github.com/cocoonstack/cocoon-common/httpx" @@ -29,9 +30,11 @@ var _ http.ResponseWriter = (*responseWriter)(nil) // Server is the Epoch HTTP server providing OCI Distribution and control plane APIs. type Server struct { - addr string // config - registryToken string // config — Bearer token for /v2/ (empty = no token required) - sso *SSOConfig // config — nil = UI auth disabled + addr string // config + registryToken string // config — Bearer token for /v2/ (empty = no token required) + sso *SSOConfig // config — nil = UI auth disabled + blobRedirect bool // config — redirect blob GETs to presigned object-store URLs + blobRedirectTTL time.Duration reg *registry.Registry // resources store *store.Store // resources @@ -54,14 +57,21 @@ func New(ctx context.Context, reg *registry.Registry, st *store.Store, addr stri if regToken != "" { logger.Info(ctx, "registry token auth enabled") } + blobRedirect, _ := strconv.ParseBool(os.Getenv("EPOCH_BLOB_REDIRECT")) + blobRedirectTTL := resolveBlobRedirectTTL(os.Getenv("EPOCH_BLOB_REDIRECT_TTL")) + if blobRedirect { + logger.Infof(ctx, "blob redirect enabled, ttl=%s", blobRedirectTTL) + } s := &Server{ - addr: addr, - registryToken: regToken, - sso: sso, - reg: reg, - store: st, - router: mux.NewRouter(), - uploads: newUploadSessions(resolveUploadDir(ctx)), + addr: addr, + registryToken: regToken, + sso: sso, + blobRedirect: blobRedirect, + blobRedirectTTL: blobRedirectTTL, + reg: reg, + store: st, + router: mux.NewRouter(), + uploads: newUploadSessions(resolveUploadDir(ctx)), } s.setupRoutes(ctx) return s From 13b78789b990ef1ee025c616c058a33ebb034c61 Mon Sep 17 00:00:00 2001 From: tonic Date: Thu, 18 Jun 2026 19:47:30 +0800 Subject: [PATCH 2/2] chore(deploy): enable blob redirect and right-size epoch-server Turn on EPOCH_BLOB_REDIRECT so pulls bypass the proxy. Bump resources from 1 CPU / 512Mi to 3 CPU / 4Gi: pushes still stream through epoch (this PR only redirects GETs) and the redirect fallback path also proxies, so the pod still needs headroom for multi-GiB transfers. --- epoch-server.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/epoch-server.yaml b/epoch-server.yaml index 9c1b374..51fcdd0 100644 --- a/epoch-server.yaml +++ b/epoch-server.yaml @@ -68,6 +68,11 @@ spec: # (tmpfs in most clusters) and OOMs on multi-GiB pushes. - name: EPOCH_UPLOAD_DIR value: /var/cache/epoch/uploads + # Serve blob GETs as redirects to presigned object-store URLs so + # multi-GiB bytes flow client<->GCS directly instead of being + # proxied (and TLS-re-encrypted) through this pod. + - name: EPOCH_BLOB_REDIRECT + value: "true" # Optional: absolute base URL clients reach the server at. Used # to anchor the OCI WWW-Authenticate realm + /v2/token. Required # when fronting epoch with a proxy that does NOT set @@ -131,11 +136,11 @@ spec: periodSeconds: 10 resources: requests: - cpu: 100m - memory: 128Mi - limits: cpu: "1" - memory: 512Mi + memory: 1Gi + limits: + cpu: "3" + memory: 4Gi volumeMounts: - name: upload-spool mountPath: /var/cache/epoch/uploads