From ef60710e0aa94f91abd73cea8843d8d2f8609677 Mon Sep 17 00:00:00 2001 From: mikkeldamsgaard Date: Tue, 23 Jun 2026 22:33:01 +0200 Subject: [PATCH] feat(netbird): dedicated gRPC/relay Services with appProtocol; bump to 0.73.2 Fixes #104: gRPC and relay (WebSocket) traffic previously shared the main server Service, which has no appProtocol. Envoy-based Gateway API controllers (Cilium, Envoy Gateway) read a Service port's appProtocol to pick the upstream codec, so routing everything through the plain Service forced an HTTP/1.1 upstream and broke gRPC (needs h2c/HTTP-2) and WebSocket (needs upgrade handling). Adds two ClusterIP Services with the same selector as the main Service: - -server-grpc with appProtocol kubernetes.io/h2c - -server-relay with appProtocol kubernetes.io/ws server.grpcRoute and server.relayHttpRoute now auto-fill omitted backendRefs to these Services. Both are enabled by default and only render when their route is enabled; set enabled=false to fall back to the main Service. Route resource names are unchanged. relayTcpRoute (raw TCP) keeps defaulting to the main Service since appProtocol does not apply. Also bumps netbird appVersion 0.72.3 -> 0.73.2 (consolidates the upstream update issues). Upstream changes are internal stability/perf and posture-check hardening; no config, env var, port, protocol, or DB migration changes. Dashboard image stays at v2.39.0. Includes unit tests for the new Services and route fallbacks, extends the Gateway API e2e to assert backendRef targets and appProtocol values, and documents the Services (including the Cilium gatewayAPI.enableAppProtocol requirement) in the README. Closes #103, #104, #110, #111, #112 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 23 +++++ charts/netbird/Chart.yaml | 2 +- charts/netbird/README.md | 97 +++++++++++++------ .../templates/server-grpc-service.yaml | 25 +++++ .../netbird/templates/server-grpcroute.yaml | 6 +- .../templates/server-relay-httproute.yaml | 6 +- .../templates/server-relay-service.yaml | 25 +++++ .../netbird/tests/server-deployment_test.yaml | 2 +- .../tests/server-grpc-service_test.yaml | 81 ++++++++++++++++ .../netbird/tests/server-grpcroute_test.yaml | 28 +++++- .../tests/server-relay-httproute_test.yaml | 40 ++++++++ .../tests/server-relay-service_test.yaml | 81 ++++++++++++++++ charts/netbird/tests/serviceaccount_test.yaml | 2 +- charts/netbird/values.yaml | 30 ++++++ ci/scripts/netbird/e2e-gateway.sh | 19 +++- 15 files changed, 429 insertions(+), 38 deletions(-) create mode 100644 charts/netbird/templates/server-grpc-service.yaml create mode 100644 charts/netbird/templates/server-relay-service.yaml create mode 100644 charts/netbird/tests/server-grpc-service_test.yaml create mode 100644 charts/netbird/tests/server-relay-service_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3605a9c..b53c894 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## Unreleased +### Added + +- **netbird**: Dedicated gRPC and relay Services with `appProtocol`, so + Envoy-based Gateway API controllers (Cilium, Envoy Gateway) configure the + correct upstream protocol. `server.grpcService` renders + `-server-grpc` with `appProtocol: kubernetes.io/h2c` (HTTP/2) and + `server.relayService` renders `-server-relay` with + `appProtocol: kubernetes.io/ws` (WebSocket). `server.grpcRoute` and + `server.relayHttpRoute` now auto-fill omitted `backendRefs` to these + Services. Both are enabled by default and only render when their route is + enabled; set `enabled: false` to fall back to the main Service. Previously + all routes shared the main Service (no `appProtocol`), which forced an + HTTP/1.1 upstream and broke gRPC and WebSocket on those controllers. + Fixes #104. + +### Changed + +- **netbird**: Bump appVersion from 0.72.3 to 0.73.2. Upstream changes are + internal (management/signal/relay stability and performance, posture-check + hardening); no config options, env vars, ports, or protocols changed, and + no database migration steps are required. Dashboard image stays at v2.39.0. + See [v0.73.2 release notes](https://github.com/netbirdio/netbird/releases/tag/v0.73.2) (#112). + ## [0.5.0] — 2026-06-11 ### Changed diff --git a/charts/netbird/Chart.yaml b/charts/netbird/Chart.yaml index ff8b23c..3c8852f 100644 --- a/charts/netbird/Chart.yaml +++ b/charts/netbird/Chart.yaml @@ -3,7 +3,7 @@ name: netbird description: A Helm chart for deploying NetBird VPN management, signal, dashboard, and relay services on Kubernetes type: application version: 0.5.0 -appVersion: "0.72.3" +appVersion: "0.73.2" keywords: - netbird - vpn diff --git a/charts/netbird/README.md b/charts/netbird/README.md index db38cbd..b5cdb3d 100644 --- a/charts/netbird/README.md +++ b/charts/netbird/README.md @@ -289,16 +289,47 @@ dashboard: - path: { type: PathPrefix, value: / } ``` -Rules that omit `backendRefs` get the netbird server / dashboard `Service` -auto-filled on port 80. Specify `backendRefs` explicitly for traffic -splitting or non-default ports. +Rules that omit `backendRefs` get a netbird server `Service` auto-filled on +port 80. The target depends on the traffic class: + +- `server.httpRoute` → the main server `Service` (`-server`). +- `server.grpcRoute` → the dedicated gRPC `Service` (`-server-grpc`). +- `server.relayHttpRoute` → the dedicated relay `Service` + (`-server-relay`). + +Specify `backendRefs` explicitly for traffic splitting or non-default ports. + +### Dedicated gRPC and relay Services (`appProtocol`) + +gRPC and relay (WebSocket) traffic cannot share the plain main `Service` +when the Gateway controller is Envoy-based (**Cilium**, **Envoy Gateway**): +those controllers read a Service port's `appProtocol` to pick the upstream +codec, and a Service with no `appProtocol` is treated as HTTP/1.1 — which +breaks gRPC (needs an HTTP/2 / h2c upstream) and WebSocket (needs upgrade +handling). The chart therefore renders two extra `ClusterIP` Services with +the same selector as the main Service but distinct `appProtocol`: + +| Service | appProtocol | Rendered when | Consumed by | +| ------------------------ | ------------------- | -------------------------- | ----------------------- | +| `-server-grpc` | `kubernetes.io/h2c` | `server.grpcRoute` on | `server.grpcRoute` | +| `-server-relay` | `kubernetes.io/ws` | `server.relayHttpRoute` on | `server.relayHttpRoute` | + +These are enabled by default (`server.grpcService.enabled`, +`server.relayService.enabled`) and only render when their route is enabled. +Set either to `false` to fall back to the main Service. You can also +reference the Services by name in your own `backendRefs`. + +> **Cilium:** set the Cilium Helm value `gatewayAPI.enableAppProtocol=true` +> so Cilium honours `appProtocol`. For Envoy Gateway, ensure the +> `EnvoyProxy`/`BackendTLSPolicy` config respects `appProtocol`. For deployments that expose relay as a raw TCP listener (no HTTP path matching), use `server.relayTcpRoute` — apiVersion `gateway.networking.k8s.io/v1alpha2`. TCPRoute ships in the Gateway API **experimental channel**; make sure its CRDs are installed (`experimental-install.yaml` from the Gateway API release) before -enabling it. +enabling it. Raw TCP carries no `appProtocol`, so `relayTcpRoute` keeps +defaulting its backendRefs to the main Service. ## STUN Networking @@ -789,31 +820,39 @@ Gateway API alternatives to the Ingress blocks above. Enabling both an Ingress and its matching route block is a template-time error. TLS is terminated at the referenced Gateway's listeners, not in these values. -| Key | Type | Default | Description | -| ----------------------------------- | ------ | ------- | ------------------------------------------------------------------------------------- | -| `server.httpRoute.enabled` | bool | `false` | Create `HTTPRoute` for HTTP (API + OAuth2). Requires `parentRefs`. | -| `server.httpRoute.parentRefs` | list | `[]` | Gateways to attach to (`name`, `namespace`, optional `sectionName`). | -| `server.httpRoute.hostnames` | list | `[]` | HTTPRoute hostnames | -| `server.httpRoute.rules` | list | `[]` | `HTTPRoute.spec.rules`. Omitted `backendRefs` default to server Service on port 80. | -| `server.httpRoute.annotations` | object | `{}` | Route annotations | -| `server.httpRoute.labels` | object | `{}` | Extra labels | -| `server.grpcRoute.enabled` | bool | `false` | Create `GRPCRoute` for Signal + Management. Works with plaintext h2c. | -| `server.grpcRoute.parentRefs` | list | `[]` | Gateway parent refs | -| `server.grpcRoute.hostnames` | list | `[]` | GRPCRoute hostnames | -| `server.grpcRoute.rules` | list | `[]` | `GRPCRoute.spec.rules` (method or header matches) | -| `server.grpcRoute.annotations` | object | `{}` | Route annotations | -| `server.grpcRoute.labels` | object | `{}` | Extra labels | -| `server.relayHttpRoute.enabled` | bool | `false` | Create `HTTPRoute` for relay + WebSocket (default Gateway API path). | -| `server.relayHttpRoute.parentRefs` | list | `[]` | Gateway parent refs | -| `server.relayHttpRoute.hostnames` | list | `[]` | HTTPRoute hostnames | -| `server.relayHttpRoute.rules` | list | `[]` | `HTTPRoute.spec.rules` | -| `server.relayHttpRoute.annotations` | object | `{}` | Route annotations | -| `server.relayHttpRoute.labels` | object | `{}` | Extra labels | -| `server.relayTcpRoute.enabled` | bool | `false` | Create `TCPRoute` (`v1alpha2`) for raw-TCP relay listeners. | -| `server.relayTcpRoute.parentRefs` | list | `[]` | Gateway parent refs | -| `server.relayTcpRoute.rules` | list | `[]` | `TCPRoute.spec.rules`. Defaults to a single rule targeting server Service on port 80. | -| `server.relayTcpRoute.annotations` | object | `{}` | Route annotations | -| `server.relayTcpRoute.labels` | object | `{}` | Extra labels | +| Key | Type | Default | Description | +| ----------------------------------- | ------ | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `server.httpRoute.enabled` | bool | `false` | Create `HTTPRoute` for HTTP (API + OAuth2). Requires `parentRefs`. | +| `server.httpRoute.parentRefs` | list | `[]` | Gateways to attach to (`name`, `namespace`, optional `sectionName`). | +| `server.httpRoute.hostnames` | list | `[]` | HTTPRoute hostnames | +| `server.httpRoute.rules` | list | `[]` | `HTTPRoute.spec.rules`. Omitted `backendRefs` default to server Service on port 80. | +| `server.httpRoute.annotations` | object | `{}` | Route annotations | +| `server.httpRoute.labels` | object | `{}` | Extra labels | +| `server.grpcRoute.enabled` | bool | `false` | Create `GRPCRoute` for Signal + Management. Works with plaintext h2c. | +| `server.grpcRoute.parentRefs` | list | `[]` | Gateway parent refs | +| `server.grpcRoute.hostnames` | list | `[]` | GRPCRoute hostnames | +| `server.grpcRoute.rules` | list | `[]` | `GRPCRoute.spec.rules` (method or header matches) | +| `server.grpcRoute.annotations` | object | `{}` | Route annotations | +| `server.grpcRoute.labels` | object | `{}` | Extra labels | +| `server.grpcService.enabled` | bool | `true` | Render a dedicated gRPC `Service` (`appProtocol: kubernetes.io/h2c`) when `grpcRoute` is on. False falls back to the main Service. | +| `server.grpcService.appProtocol` | string | `kubernetes.io/h2c` | `appProtocol` on the gRPC Service port. | +| `server.grpcService.port` | int | `80` | gRPC Service port (targets the `http` container port). | +| `server.grpcService.annotations` | object | `{}` | gRPC Service annotations | +| `server.relayHttpRoute.enabled` | bool | `false` | Create `HTTPRoute` for relay + WebSocket (default Gateway API path). | +| `server.relayHttpRoute.parentRefs` | list | `[]` | Gateway parent refs | +| `server.relayHttpRoute.hostnames` | list | `[]` | HTTPRoute hostnames | +| `server.relayHttpRoute.rules` | list | `[]` | `HTTPRoute.spec.rules` | +| `server.relayHttpRoute.annotations` | object | `{}` | Route annotations | +| `server.relayHttpRoute.labels` | object | `{}` | Extra labels | +| `server.relayService.enabled` | bool | `true` | Render a dedicated relay `Service` (`appProtocol: kubernetes.io/ws`) when `relayHttpRoute` is on. False falls back to the main Service. | +| `server.relayService.appProtocol` | string | `kubernetes.io/ws` | `appProtocol` on the relay Service port. | +| `server.relayService.port` | int | `80` | Relay Service port (targets the `http` container port). | +| `server.relayService.annotations` | object | `{}` | Relay Service annotations | +| `server.relayTcpRoute.enabled` | bool | `false` | Create `TCPRoute` (`v1alpha2`) for raw-TCP relay listeners. | +| `server.relayTcpRoute.parentRefs` | list | `[]` | Gateway parent refs | +| `server.relayTcpRoute.rules` | list | `[]` | `TCPRoute.spec.rules`. Defaults to a single rule targeting server Service on port 80. | +| `server.relayTcpRoute.annotations` | object | `{}` | Route annotations | +| `server.relayTcpRoute.labels` | object | `{}` | Extra labels | #### Server Pod diff --git a/charts/netbird/templates/server-grpc-service.yaml b/charts/netbird/templates/server-grpc-service.yaml new file mode 100644 index 0000000..acc0f73 --- /dev/null +++ b/charts/netbird/templates/server-grpc-service.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.server.grpcService.enabled .Values.server.grpcRoute.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "netbird.server.fullname" . }}-grpc + namespace: {{ .Release.Namespace }} + labels: + {{- include "netbird.server.labels" . | nindent 4 }} + {{- with .Values.server.grpcService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: grpc + port: {{ .Values.server.grpcService.port }} + targetPort: http + protocol: TCP + {{- with .Values.server.grpcService.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "netbird.server.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/netbird/templates/server-grpcroute.yaml b/charts/netbird/templates/server-grpcroute.yaml index d85b795..984d8c0 100644 --- a/charts/netbird/templates/server-grpcroute.yaml +++ b/charts/netbird/templates/server-grpcroute.yaml @@ -1,10 +1,14 @@ {{- if .Values.server.grpcRoute.enabled }} {{- $serviceName := include "netbird.server.fullname" . }} +{{- $backendName := $serviceName }} +{{- if .Values.server.grpcService.enabled }} + {{- $backendName = printf "%s-grpc" $serviceName }} +{{- end }} {{- $rules := list }} {{- range .Values.server.grpcRoute.rules }} {{- $rule := deepCopy . }} {{- if not (hasKey $rule "backendRefs") }} - {{- $_ := set $rule "backendRefs" (list (dict "name" $serviceName "port" 80)) }} + {{- $_ := set $rule "backendRefs" (list (dict "name" $backendName "port" 80)) }} {{- end }} {{- $rules = append $rules $rule }} {{- end }} diff --git a/charts/netbird/templates/server-relay-httproute.yaml b/charts/netbird/templates/server-relay-httproute.yaml index 415df79..bffd239 100644 --- a/charts/netbird/templates/server-relay-httproute.yaml +++ b/charts/netbird/templates/server-relay-httproute.yaml @@ -1,10 +1,14 @@ {{- if .Values.server.relayHttpRoute.enabled }} {{- $serviceName := include "netbird.server.fullname" . }} +{{- $backendName := $serviceName }} +{{- if .Values.server.relayService.enabled }} + {{- $backendName = printf "%s-relay" $serviceName }} +{{- end }} {{- $rules := list }} {{- range .Values.server.relayHttpRoute.rules }} {{- $rule := deepCopy . }} {{- if not (hasKey $rule "backendRefs") }} - {{- $_ := set $rule "backendRefs" (list (dict "name" $serviceName "port" 80)) }} + {{- $_ := set $rule "backendRefs" (list (dict "name" $backendName "port" 80)) }} {{- end }} {{- $rules = append $rules $rule }} {{- end }} diff --git a/charts/netbird/templates/server-relay-service.yaml b/charts/netbird/templates/server-relay-service.yaml new file mode 100644 index 0000000..77129ec --- /dev/null +++ b/charts/netbird/templates/server-relay-service.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.server.relayService.enabled .Values.server.relayHttpRoute.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "netbird.server.fullname" . }}-relay + namespace: {{ .Release.Namespace }} + labels: + {{- include "netbird.server.labels" . | nindent 4 }} + {{- with .Values.server.relayService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: relay + port: {{ .Values.server.relayService.port }} + targetPort: http + protocol: TCP + {{- with .Values.server.relayService.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "netbird.server.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/netbird/tests/server-deployment_test.yaml b/charts/netbird/tests/server-deployment_test.yaml index 3f33cf8..8caf00f 100644 --- a/charts/netbird/tests/server-deployment_test.yaml +++ b/charts/netbird/tests/server-deployment_test.yaml @@ -13,7 +13,7 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: "netbirdio/netbird-server:0.72.3" + value: "netbirdio/netbird-server:0.73.2" - it: should use custom image tag when set set: diff --git a/charts/netbird/tests/server-grpc-service_test.yaml b/charts/netbird/tests/server-grpc-service_test.yaml new file mode 100644 index 0000000..c06e713 --- /dev/null +++ b/charts/netbird/tests/server-grpc-service_test.yaml @@ -0,0 +1,81 @@ +suite: server gRPC service tests +templates: + - templates/server-grpc-service.yaml +tests: + - it: should not render when grpcRoute is disabled + asserts: + - hasDocuments: + count: 0 + + - it: should not render when grpcService is disabled even if grpcRoute is enabled + set: + server.grpcRoute.enabled: true + server.grpcService.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should render when grpcRoute is enabled + set: + server.grpcRoute.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service + + - it: should have -grpc suffix in name + set: + server.grpcRoute.enabled: true + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-netbird-server-grpc + + - it: should carry the h2c appProtocol by default + set: + server.grpcRoute.enabled: true + asserts: + - contains: + path: spec.ports + content: + name: grpc + port: 80 + targetPort: http + protocol: TCP + appProtocol: kubernetes.io/h2c + + - it: should allow overriding appProtocol and port + set: + server.grpcRoute.enabled: true + server.grpcService.appProtocol: custom/h2c + server.grpcService.port: 8443 + asserts: + - contains: + path: spec.ports + content: + name: grpc + port: 8443 + targetPort: http + protocol: TCP + appProtocol: custom/h2c + + - it: should share the server selector labels + set: + server.grpcRoute.enabled: true + asserts: + - isSubset: + path: spec.selector + content: + app.kubernetes.io/name: netbird + app.kubernetes.io/component: server + + - it: should apply custom annotations + set: + server.grpcRoute.enabled: true + server.grpcService.annotations: + example.com/foo: bar + asserts: + - equal: + path: metadata.annotations["example.com/foo"] + value: bar diff --git a/charts/netbird/tests/server-grpcroute_test.yaml b/charts/netbird/tests/server-grpcroute_test.yaml index 6fcae81..e57bbae 100644 --- a/charts/netbird/tests/server-grpcroute_test.yaml +++ b/charts/netbird/tests/server-grpcroute_test.yaml @@ -51,7 +51,7 @@ tests: path: spec.rules[1].matches[0].method.service value: management.ManagementService - - it: should auto-fill backendRefs when omitted + - it: should auto-fill backendRefs to the dedicated gRPC service when omitted set: server.grpcRoute.enabled: true server.grpcRoute.parentRefs: @@ -62,11 +62,35 @@ tests: asserts: - equal: path: spec.rules[0].backendRefs[0].name - value: RELEASE-NAME-netbird-server + value: RELEASE-NAME-netbird-server-grpc - equal: path: spec.rules[0].backendRefs[0].port value: 80 + - it: should fall back to the main service when grpcService is disabled + set: + server.grpcRoute.enabled: true + server.grpcService.enabled: false + server.grpcRoute.parentRefs: + - name: gw + server.grpcRoute.rules: + - matches: + - method: { service: signalexchange.SignalExchange } + asserts: + - equal: + path: spec.rules[0].backendRefs[0].name + value: RELEASE-NAME-netbird-server + + - it: should keep the route name unchanged when grpcService is enabled + set: + server.grpcRoute.enabled: true + server.grpcRoute.parentRefs: + - name: gw + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-netbird-server-grpc + - it: should support header-based matches (traefik-style) set: server.grpcRoute.enabled: true diff --git a/charts/netbird/tests/server-relay-httproute_test.yaml b/charts/netbird/tests/server-relay-httproute_test.yaml index a241cdd..25f6ede 100644 --- a/charts/netbird/tests/server-relay-httproute_test.yaml +++ b/charts/netbird/tests/server-relay-httproute_test.yaml @@ -47,3 +47,43 @@ tests: - equal: path: spec.rules[0].matches[1].path.value value: /ws-proxy + + - it: should auto-fill backendRefs to the dedicated relay service when omitted + set: + server.relayHttpRoute.enabled: true + server.relayHttpRoute.parentRefs: + - name: gw + server.relayHttpRoute.rules: + - matches: + - path: { type: PathPrefix, value: /relay } + asserts: + - equal: + path: spec.rules[0].backendRefs[0].name + value: RELEASE-NAME-netbird-server-relay + - equal: + path: spec.rules[0].backendRefs[0].port + value: 80 + + - it: should fall back to the main service when relayService is disabled + set: + server.relayHttpRoute.enabled: true + server.relayService.enabled: false + server.relayHttpRoute.parentRefs: + - name: gw + server.relayHttpRoute.rules: + - matches: + - path: { type: PathPrefix, value: /relay } + asserts: + - equal: + path: spec.rules[0].backendRefs[0].name + value: RELEASE-NAME-netbird-server + + - it: should keep the route name unchanged when relayService is enabled + set: + server.relayHttpRoute.enabled: true + server.relayHttpRoute.parentRefs: + - name: gw + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-netbird-server-relay diff --git a/charts/netbird/tests/server-relay-service_test.yaml b/charts/netbird/tests/server-relay-service_test.yaml new file mode 100644 index 0000000..d9b940a --- /dev/null +++ b/charts/netbird/tests/server-relay-service_test.yaml @@ -0,0 +1,81 @@ +suite: server relay service tests +templates: + - templates/server-relay-service.yaml +tests: + - it: should not render when relayHttpRoute is disabled + asserts: + - hasDocuments: + count: 0 + + - it: should not render when relayService is disabled even if relayHttpRoute is enabled + set: + server.relayHttpRoute.enabled: true + server.relayService.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: should render when relayHttpRoute is enabled + set: + server.relayHttpRoute.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service + + - it: should have -relay suffix in name + set: + server.relayHttpRoute.enabled: true + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-netbird-server-relay + + - it: should carry the ws appProtocol by default + set: + server.relayHttpRoute.enabled: true + asserts: + - contains: + path: spec.ports + content: + name: relay + port: 80 + targetPort: http + protocol: TCP + appProtocol: kubernetes.io/ws + + - it: should allow overriding appProtocol and port + set: + server.relayHttpRoute.enabled: true + server.relayService.appProtocol: custom/ws + server.relayService.port: 8080 + asserts: + - contains: + path: spec.ports + content: + name: relay + port: 8080 + targetPort: http + protocol: TCP + appProtocol: custom/ws + + - it: should share the server selector labels + set: + server.relayHttpRoute.enabled: true + asserts: + - isSubset: + path: spec.selector + content: + app.kubernetes.io/name: netbird + app.kubernetes.io/component: server + + - it: should apply custom annotations + set: + server.relayHttpRoute.enabled: true + server.relayService.annotations: + example.com/foo: bar + asserts: + - equal: + path: metadata.annotations["example.com/foo"] + value: bar diff --git a/charts/netbird/tests/serviceaccount_test.yaml b/charts/netbird/tests/serviceaccount_test.yaml index 4e12a0b..e820c50 100644 --- a/charts/netbird/tests/serviceaccount_test.yaml +++ b/charts/netbird/tests/serviceaccount_test.yaml @@ -64,4 +64,4 @@ tests: content: helm.sh/chart: netbird-0.5.0 app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: "0.72.3" + app.kubernetes.io/version: "0.73.2" diff --git a/charts/netbird/values.yaml b/charts/netbird/values.yaml index 62e5313..333f011 100644 --- a/charts/netbird/values.yaml +++ b/charts/netbird/values.yaml @@ -551,6 +551,22 @@ server: annotations: {} labels: {} + # -- Dedicated ClusterIP Service for gRPC traffic, carrying + # `appProtocol: kubernetes.io/h2c`. Envoy-based Gateway API controllers + # (Cilium, Envoy Gateway) read appProtocol to configure an HTTP/2 (h2c) + # upstream cluster; routing gRPC through the main Service (which has no + # appProtocol) makes them use a plain HTTP/1.1 upstream, which breaks gRPC. + # Rendered only when grpcRoute is enabled; grpcRoute auto-fills its + # backendRefs to this Service. Set enabled=false to fall back to the main + # Service. The Service is named "-server-grpc". + # When using Cilium, also set the Cilium Helm value + # gatewayAPI.enableAppProtocol=true so the appProtocol is honored. + grpcService: + enabled: true + appProtocol: kubernetes.io/h2c + port: 80 + annotations: {} + # -- Ingress for relay and WebSocket routes. # Mutually exclusive with server.relayHttpRoute and server.relayTcpRoute. ingressRelay: @@ -575,6 +591,20 @@ server: annotations: {} labels: {} + # -- Dedicated ClusterIP Service for relay/WebSocket traffic, carrying + # `appProtocol: kubernetes.io/ws`. Envoy-based Gateway API controllers read + # appProtocol to enable WebSocket upgrade handling on the upstream cluster; + # routing WebSocket through the main Service (no appProtocol) makes them use + # a plain HTTP/1.1 upstream without upgrade support, which breaks the relay. + # Rendered only when relayHttpRoute is enabled; relayHttpRoute auto-fills its + # backendRefs to this Service. Set enabled=false to fall back to the main + # Service. The Service is named "-server-relay". + relayService: + enabled: true + appProtocol: kubernetes.io/ws + port: 80 + annotations: {} + # -- Gateway API TCPRoute for relay exposed as raw TCP. # Alternative to relayHttpRoute for deployments that front relay with a # dedicated TCP listener (no HTTP path matching). TCPRoute is v1alpha2. diff --git a/ci/scripts/netbird/e2e-gateway.sh b/ci/scripts/netbird/e2e-gateway.sh index 52151d4..47f57eb 100755 --- a/ci/scripts/netbird/e2e-gateway.sh +++ b/ci/scripts/netbird/e2e-gateway.sh @@ -162,9 +162,24 @@ assert_backend_ref() { } assert_backend_ref httproute "$RELEASE-server" "$RELEASE-server" -assert_backend_ref httproute "$RELEASE-server-relay" "$RELEASE-server" +assert_backend_ref httproute "$RELEASE-server-relay" "$RELEASE-server-relay" assert_backend_ref httproute "$RELEASE-dashboard" "$RELEASE-dashboard" -assert_backend_ref grpcroute "$RELEASE-server-grpc" "$RELEASE-server" +assert_backend_ref grpcroute "$RELEASE-server-grpc" "$RELEASE-server-grpc" + +# ── Confirm dedicated gRPC/relay Services exist with appProtocol ────── +assert_service_app_protocol() { + local name="$1" expected_proto="$2" + local proto + proto=$(kubectl -n "$NAMESPACE" get service "$name" \ + -o jsonpath='{.spec.ports[0].appProtocol}') + if [ "$proto" != "$expected_proto" ]; then + fail "service/$name ports[0].appProtocol = '$proto' (expected '$expected_proto')" + fi + log " service/$name appProtocol = $proto ✓" +} + +assert_service_app_protocol "$RELEASE-server-grpc" "kubernetes.io/h2c" +assert_service_app_protocol "$RELEASE-server-relay" "kubernetes.io/ws" # ── Confirm mutual-exclusion validation trips ──────────────────────── log "Verifying template validation: enabling Ingress + HTTPRoute should fail..."