Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion config.example/group_vars/k8s_cluster.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,10 @@ dashboard_metrics_scraper_repo: "kubernetesui/metrics-scraper"
# Ensure hosts file generation only runs across k8s cluster
hosts_add_ansible_managed_hosts_groups: ["k8s_cluster"]

# NFS Client Provisioner
# NFS Client Provisioner
# Playbook: nfs-client-provisioner.yml
# This convenience default creates a non-HA NFS export on the first control-plane
# node. For production clusters, point this at site-owned storage or disable it.
k8s_nfs_client_provisioner: true
k8s_deploy_nfs_server: true
k8s_nfs_mkdir: true # Set to false if an export dir is already configured with proper permissions
Expand Down
62 changes: 34 additions & 28 deletions config.example/group_vars/netapp-trident.yml
Original file line number Diff line number Diff line change
@@ -1,29 +1,36 @@
---
# vars file for netapp-trident playbook

# URL of the Trident installer package that you wish to download and use
trident_version: "22.01.0"
# Version of the Trident installer package to download when tridentctl is needed
trident_version: "26.02.1"
trident_installer_url: "https://github.com/NetApp/trident/releases/download/v{{ trident_version }}/trident-installer-{{ trident_version }}.tar.gz"

# Official Trident Helm chart settings
trident_helm_repo_name: netapp-trident
trident_helm_repo_url: https://netapp.github.io/trident-helm-chart
trident_helm_release_name: trident-operator
trident_helm_chart_name: "{{ trident_helm_repo_name }}/trident-operator"
trident_helm_chart_version: "100.2602.1"

# Namespace to install Trident in
trident_namespace: deepops-trident

# Denotes whether or not to create new backends after deploying trident
# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operator-install.html#creating-a-trident-backend
create_backends: true
# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/ontap-nas-examples.html
create_backends: false
trident_hide_backend_details: true

# List of backends to create
# For more info on parameter values, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operations/tasks/backends/ontap.html
# Note: Parameters other than those listed below are not avaible when creating a backend via DeepOps
# For more info on parameter values, refer to: https://docs.netapp.com/us-en/trident/trident-use/ontap-nas-examples.html
# Note: Parameters other than those listed below are not available when creating a backend via DeepOps
# If you wish to use other parameter values, you must create your backend manually.
backends_to_create:
- backendName: ontap-flexvol
storageDriverName: ontap-nas # only 'ontap-nas' and 'ontap-nas-flexgroup' are supported when creating a backend via DeepOps
managementLIF: 10.61.188.40
dataLIF: 192.168.200.41
svm: ai221_data
username: admin
password: NetApp!23
managementLIF: ontap-mgmt.example.com
dataLIF: ontap-nfs.example.com
svm: svm_nfs
username: vsadmin
storagePrefix: trident
limitAggregateUsage: ""
limitVolumeSize: ""
Expand All @@ -41,11 +48,10 @@ backends_to_create:
tieringPolicy: none
- backendName: ontap-flexgroup
storageDriverName: ontap-nas-flexgroup # only 'ontap-nas' and 'ontap-nas-flexgroup' are supported when creating a backend via DeepOps
managementLIF: 10.61.188.40
dataLIF: 192.168.200.41
svm: ai221_data
username: admin
password: NetApp!23
managementLIF: ontap-mgmt.example.com
dataLIF: ontap-nfs.example.com
svm: svm_nfs
username: vsadmin
storagePrefix: trident
limitAggregateUsage: ""
limitVolumeSize: ""
Expand All @@ -64,12 +70,12 @@ backends_to_create:
# Add additional backends as needed

# Denotes whether or not to create new StorageClasses for your NetApp storage
# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operator-install.html#creating-a-storage-class
create_StorageClasses: true
# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/create-stor-class.html
create_StorageClasses: false

# List of StorageClasses to create
# Note: Each item in the list should be an actual K8s StorageClass definition in yaml format
# For more info on StorageClass definitions, refer to https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/concepts/objects.html#kubernetes-storageclass-objects.
# For more info on StorageClass definitions, refer to https://docs.netapp.com/us-en/trident/trident-use/create-stor-class.html
storageClasses_to_create:
- apiVersion: storage.k8s.io/v1
kind: StorageClass
Expand All @@ -90,30 +96,30 @@ storageClasses_to_create:
# Add additional StorageClasses as needed

# Denotes whether or not to enable volume snapshots in your Kubernetes cluster
# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v21.01/kubernetes/operations/tasks/volumes/snapshots.html
enable_volume_snapshots: true
# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/vol-snapshots.html
enable_volume_snapshots: false

# List of volume snapshot CRDs to create
volume_snapshot_CRD_manifest_URLs:
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml

# List of snapshot controller manifest URLs
snapshot_controller_manifest_URLs:
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
- https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml

# List of volume snapshot classes to create
volume_snapshot_classes_to_create:
- apiVersion: snapshot.storage.k8s.io/v1beta1
- apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
name: csi-snapclass
driver: csi.trident.netapp.io
deletionPolicy: Delete

# Denotes whether or not to copy tridenctl binary to localhost
# Denotes whether or not to copy tridentctl binary to localhost
copy_tridentctl_to_localhost: true
# Directory that tridentctl will be copied to on localhost
tridentctl_copy_to_directory: ../../ # will be copied to 'deepops/' directory
2 changes: 1 addition & 1 deletion docs/deepops/proxy-k8s-cluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,4 @@ Before executing the scripts to continue setting up and installing services you
1. Run the `scripts/deepops/proxy.sh` script to setup environment variables. Then continue running all the scripts necessary for your environment.

2. Rather than setup the variables for your current shell, you can run each script to use the proxy without impacting your current env. For example:
`. scripts/deepops/proxy.sh && scripts/k8s/deploy_rook.sh`
`. scripts/deepops/proxy.sh && DEEPOPS_ENABLE_DEPRECATED_ROOK=true scripts/k8s/deploy_rook.sh`
27 changes: 16 additions & 11 deletions docs/k8s-cluster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Instructions for deploying a GPU cluster with Kubernetes
- [Persistent Storage](#persistent-storage)
- [NFS Client Provisioner](#nfs-client-provisioner)
- [Ceph Cluster (deprecated)](#ceph-cluster-deprecated)
- [NetApp Astra Trident](#netapp-astra-trident)
- [NetApp Trident](#netapp-trident)
- [Monitoring](#monitoring)
- [Logging](#logging)
- [Centralized syslog](#centralized-syslog)
Expand Down Expand Up @@ -124,23 +124,26 @@ Run the following script to create an administrative user and print out the dash

#### NFS Client Provisioner

The default behavior of DeepOps is to setup an NFS server on the first `kube_control_plane` node. This temporary NFS server is used by the `nfs-client-provisioner` which is installed as the default StorageClass of a standard DeepOps deployment.
The default DeepOps Kubernetes storage path is the `nfs-subdir-external-provisioner` chart, backed by an NFS export on the first `kube_control_plane` node. This is a convenience default for examples, smoke tests, and small internal clusters. It is not a highly available production storage design.

To use an existing nfs server server update the `k8s_nfs_server` and `k8s_nfs_export_path` variables in `config/group_vars/k8s-cluster.yml` and set the `k8s_deploy_nfs_server` to false in `config/group_vars/k8s-cluster.yml`. Additionally, the `k8s_nfs_mkdir` variable can be set to `false` if the export directory is already configured on the server.
To use an existing NFS server, update the `k8s_nfs_server` and `k8s_nfs_export_path` variables in `config/group_vars/k8s-cluster.yml` and set `k8s_deploy_nfs_server` to `false`. Additionally, set `k8s_nfs_mkdir` to `false` if the export directory is already configured on the server.

To manually install or re-install the `nfs-client-provisioner` run:

```bash
ansible-playbook playbooks/k8s-cluster/nfs-client-provisioner.yml
```

To skip this installation set `k8s_nfs_client_provisioner` to `false`.
To skip this installation, set `k8s_nfs_client_provisioner` to `false`.

#### Ceph Cluster (deprecated)

For a non-nfs based alternative, deploy a Ceph cluster running on Kubernetes for services that require persistent storage (such as Kubeflow):
The Rook/Ceph helper is deprecated and community-supported. It predates the current Rook Helm chart layout and should not be used as a new production storage path without a site-owned Rook/Ceph design and validation plan.

For legacy environments that still use this helper, deploy a Ceph cluster running on Kubernetes with:

```bash
export DEEPOPS_ENABLE_DEPRECATED_ROOK=true
./scripts/k8s/deploy_rook.sh
```

Expand All @@ -150,17 +153,19 @@ Poll the Ceph status by running (this script will return when Ceph initializatio
./scripts/k8s/deploy_rook.sh -w
```

#### NetApp Astra Trident
#### NetApp Trident

Deploy NetApp Astra Trident for services that require persistent storage (such as Kubeflow). Note that you must have a supported NetApp storage system/instance/service in order to use Astra Trident to provision persistent storage.
The Trident role is optional and community-supported in DeepOps. Use it only when the site already has a supported NetApp storage system or service and a storage owner who can validate the backend, StorageClass, and snapshot policy against current NetApp documentation.

1. Set configuration parameters.

```bash
vi config/group_vars/netapp-trident.yml
```

2. Deploy Astra Trident using Ansible.
By default, the example configuration installs Trident but does not create a backend, StorageClass, or snapshot controller. Set `create_backends`, `create_StorageClasses`, or `enable_volume_snapshots` to `true` only after replacing the example backend values with site-specific storage details and auth values.

2. Deploy Trident using Ansible.

```bash
# NOTE: If SSH requires a password, add: `-k`
Expand All @@ -169,7 +174,7 @@ Deploy NetApp Astra Trident for services that require persistent storage (such a
ansible-playbook -l k8s_cluster playbooks/k8s-cluster/netapp-trident.yml
```

3. Verify that Astra Trident is running.
3. Verify that Trident is running.

```bash
./tridentctl -n deepops-trident version
Expand All @@ -181,11 +186,11 @@ Deploy NetApp Astra Trident for services that require persistent storage (such a
+----------------+----------------+
| SERVER VERSION | CLIENT VERSION |
+----------------+----------------+
| 22.01.0 | 22.01.0 |
| 26.02.1 | 26.02.1 |
+----------------+----------------+
```

For more information on Astra Trident, please refer to the [official documentation](https://docs.netapp.com/us-en/trident/index.html).
For more information on Trident, refer to the [official documentation](https://docs.netapp.com/us-en/trident/index.html).

### Monitoring

Expand Down
4 changes: 2 additions & 2 deletions docs/k8s-cluster/kubeflow.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ As part of the Kubeflow installation, the MPI Operator will also be installed. T

Deploy Kubernetes by following the [DeepOps Kubernetes Deployment Guide](README.md)

Kubeflow requires a DefaultStorageClass to be defined. By default DeepOps installs the `nfs-client-provisioner` using the [nfs-client-provisioner.yml playbook](../../playbooks/k8s-cluster/nfs-client-provisioner.yml). This playbook can re run manually. As an NFS alternative [Ceph](../../scripts/k8s/deploy_rook.sh), [Trident](../../playbooks/k8s-cluster/netapp-trident.yml) or an alternative StorageClass can be used.
Kubeflow requires a DefaultStorageClass to be defined. By default DeepOps installs the `nfs-client-provisioner` using the [nfs-client-provisioner.yml playbook](../../playbooks/k8s-cluster/nfs-client-provisioner.yml). This playbook can be rerun manually. For production or HA storage, use a site-owned StorageClass. The legacy [Ceph](../../scripts/k8s/deploy_rook.sh) helper is deprecated, and [Trident](../../playbooks/k8s-cluster/netapp-trident.yml) is optional and community-supported unless a site storage owner validates it.

Deploy Kubeflow:

Expand Down Expand Up @@ -125,6 +125,6 @@ To correct this issue:

1. Uninstall Rook/Ceph: `./scripts/k8s/deploy_rook.sh -d`
2. Uninstall Kubeflow: `./scripts/k8s/deploy_kubeflow.sh -d`
3. Re-install Rook/ceph: `./scripts/k8s/deploy_rook.sh`
3. Re-install Rook/Ceph: `DEEPOPS_ENABLE_DEPRECATED_ROOK=true ./scripts/k8s/deploy_rook.sh`
4. Poll for Ceph to initialize (wait for this script to exit): `./scripts/k8s/deploy_rook.sh -w`
5. Re-install Kubeflow: `./scripts/k8s/deploy_kubeflow.sh`
18 changes: 11 additions & 7 deletions playbooks/k8s-cluster/nfs-client-provisioner.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
---
# This playbook is used to setup the nfs-client-provisioner
# If the cluster already has an nfs server setup run with --skip-tags="nfs_server,nfs_mkdir"
# Otherwhise an NFS server will be configured on kube_control_plane[0]
# After settting up an NFS server, nfs software is installed on all nodes
# Lastly, the nfs-client-provisioner is helm installed using the role
# Otherwise an NFS server will be configured on kube_control_plane[0]
# After setting up an NFS server, NFS software is installed on all nodes.
# Lastly, the nfs-client-provisioner chart is installed using the role.
#
# The default NFS configuration of this playbook is not meant for production systems
# For production deployments please configur NFS with storage-partner guidance
# The default NFS configuration of this playbook is not meant for production systems.
# For production deployments, configure storage with storage-partner guidance.
#
# Install the required NFS software on all nodes and then helm install the nfs-client
# Install the required NFS software on all nodes and then helm install the NFS client.
- hosts: kube_control_plane[0]
become: yes
tasks:
Expand All @@ -17,13 +17,17 @@
path: "{{ k8s_nfs_export_path }}"
state: directory
mode: "u=rwx,g=rwx,o=rwx"
when: k8s_nfs_mkdir and k8s_deploy_nfs_server
when: k8s_nfs_mkdir and k8s_deploy_nfs_server
tags:
- nfs_mkdir
- name: Deploy NFS server master
include_role:
name: nfs
vars:
nfs_is_server: yes
when: k8s_deploy_nfs_server
tags:
- nfs_server

- hosts: "k8s_cluster"
become: yes
Expand Down
4 changes: 3 additions & 1 deletion roles/netapp-trident/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ netapp-trident

Ansible role that can be used to deploy NetApp Trident within a Kubernetes cluster.

This role is optional and community-supported in DeepOps. Use it only when a site storage owner can validate the NetApp backend, StorageClass, and snapshot policy against current NetApp documentation.

Requirements
------------

Expand All @@ -15,7 +17,7 @@ Requirements
Role Variables
--------------

See defaults/main.yml, vars/main.yml
See defaults/main.yml and `config.example/group_vars/netapp-trident.yml`.

Dependencies
------------
Expand Down
Loading
Loading