Skip to content

Kustomization - kubectl kustomize

Declarative Configuration in Kubernetes

  • Kustomize is a very powerful too for customizing and building Kubernetes resources.
  • Kustomize started at 2017, and added to kubectl since version 1.14.
  • Kustomize has many useful features for managing and deploying resource.
  • When you execute a Kustomization beside using the builtin features, it will also re-order the resources in a logical way for the K8S to be deployed.

01. Re-order the resources

  • Kustomization re-orders the Kind for optimization. For this demo, we will need an existing namespace before using it.

  • The order of the resources is defined in the source code

// An attempt to order things to help k8s, e.g.
// - Namespace should be first.
// - Service should come before things that refer to it.
// In some cases order just specified to provide determinism.
var orderFirst = []string{
    "Namespace",
    "ResourceQuota",
    "StorageClass",
    "CustomResourceDefinition",
    "ServiceAccount",
    "PodSecurityPolicy",
    "Role",
    "ClusterRole",
    "RoleBinding",
    "ClusterRoleBinding",
    "ConfigMap",
    "Secret",
    "Endpoints",
    "Service",
    "LimitRange",
    "PriorityClass",
    "PersistentVolume",
    "PersistentVolumeClaim",
    "Deployment",
    "StatefulSet",
    "CronJob",
    "PodDisruptionBudget",
}

var orderLast = []string{
    "MutatingWebhookConfiguration",
    "ValidatingWebhookConfiguration",
}

02. Base resource for our demo

  • In the following samples we will refer to the following base.yaml file:
# base.yaml
# This is the base file for all the demos in this folder
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
        - name: myapp
          image: __image__

03. Common Features

commonAnnotation

kubectl kustomize samples/01-commonAnnotation

### FileName: kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

# This will add annotation under every metadata entry
# ex: main metadata, spec.metadata etc
commonAnnotations:
  author: nirgeier@gmail.com
  • Output:
### commonAnnotation output
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    ### Annotation added here
    author: nirgeier@gmail.com
    name: myapp
spec:
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      ### Annotation added here
      annotations:
        author: nirgeier@gmail.com
      labels:
        app: myapp
    spec:
      containers:
        - image: __image__
          name: myapp

commonLabels

kubectl kustomize samples/02-commonLabels

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

# This will add annotation under every metadata entry
# ex: main metadata, spec.metadata etc
commonLabels:
  author: nirgeier@gmail.com
  env: codeWizard-cluster

bases:
  - ../_base
  • Output:
apiVersion: apps/v1
kind: Deployment
metadata:
    # Labels added ....
    labels:
    author: nirgeier@gmail.com
    env: codeWizard-cluster
  name: myapp
spec:
  selector:
    matchLabels:
      app: myapp
      # Labels added ....
      author: nirgeier@gmail.com
      env: codeWizard-cluster
  template:
    metadata:
      labels:
        app: myapp
        # Labels added ....
        author: nirgeier@gmail.com
        env: codeWizard-cluster
    spec:
      containers:
      - image: __image__
        name: myapp

Generators

  • Kustomization also support generate ConfigMap / Secret in several ways.
  • The default behavior is adding the output hash value as suffix to the name, e.g.: secretMapFromFile-495dtcb64g
apiVersion: v1
data:
  APP_ENV: ZGV2ZWxvcG1lbnQ=
  LOG_DEBUG: dHJ1ZQ==
  NODE_ENV: ZGV2
  REGION: d2V1
kind: Secret
metadata:
  name: secretMapFromFile-495dtcb64g # <--------------------------
type: Opaque
  • We can disable the suffix with the following addition to the kustomization.yaml
generatorOptions:
  disableNameSuffixHash: true

configMapGenerator

  • From Env

    • .env 
      key1=value1
env=qa

    • kustomization.yaml 

      # Generate config file from env file
configMapGenerator:
  - name: configMapFromEnv
    env: .env

    • The output of configMapFromEnv: 

      apiVersion: v1
data:
  env: qa
  key1: value1
kind: ConfigMap
metadata:
  name: configMapFromEnv-c9655hf97k

  • From File

    • .env 
      key1=value1
env=qa

    • kustomization.yaml 

      # Generate config file from env file
configMapGenerator:
  - name: configMapFromEnv
    files: 
    - .env

    • The output of configMapFromEnv: 

      apiVersion: v1
data:
  .env: "key1=value1\r\nenv=qa" # <--------------------------
kind: ConfigMap
metadata:
  name: configFromFile-dfhmctd84d

  • From Literal

    • .env 
      key1=value1
env=qa

    • kustomization.yaml 

      configMapGenerator:
  - name: configFromLiterals
    literals:
      - Key1=value1
      - Key2=value2

    • The output of configMapFromEnv: 

      apiVersion: v1
data:
  Key1: value1
  Key2: value2
kind: ConfigMap
metadata:
  name: configFromLiterals-h777b4gdf5

Secret Generator

# Similar to configMap but with an additional type field
secretGenerator:
  # Generate secret from env file
  - name: secretMapFromFile
    env: .env
    type: Opaque
generatorOptions:
  disableNameSuffixHash: true

images

  • Modify the name, tags and/or digest for images.
kubectl kustomize samples/04-images

# kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
  - ./base.yaml

images:
  # The image as its defined in the Deployment file
  - name: __image__
    # The new name to set
    newName: my-registry/my-image
    # optional: image tag
    newTag: v1
  • Output:
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
        # --- This image was updated
        - image: my-registry/my-image:v1
          name: myapp

Namespaces

kubectl kustomize samples/05-Namespace

# kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

# Add the desired namespace to all resources
namespace: kustomize-namespace

bases:
  - ../_base
  • Output:
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
  # Namespace added here
  namespace: kustomize-namespace

Prefix-suffix

kubectl kustomize samples/06-Prefix-Suffix

# kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

# Add the desired Prefix to all resources
namePrefix: prefix-codeWizard-
nameSuffix: -suffix-codeWizard

bases:
  - ../_base
  • Output:
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prefix-codeWizard-myapp-suffix-codeWizard

Replicas

  • deployment
# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: deployment
spec:
  replicas: 5
  selector:
    name: deployment
  template:
    containers:
      - name: container
        image: registry/conatiner:latest
  • kustomization
# kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

replicas:
  - name: deployment
    count: 10

resources:
  - deployment.yaml
  • Output:

Note

There is a bug with the replicas entries which return error for some reason.

$ kubectl kustomize .

# For some reason we get this error:
Error: json: unknown field "replicas"

# Workaround for this error for now is:
$ kustomize build .

apiVersion: apps/v1
kind: Deployment
metadata:
  name: deployment
spec:
  replicas: 10
  selector:
    name: deployment
  template:
    containers:
      - image: registry/conatiner:latest
        name: container

Patches

  • There are several types of patches like [replace, delete, patchesStrategicMerge]
  • For this demo we will demonstrate patchesStrategicMerge

Patch Add/Update

kubectl kustomize samples/08-Patches/patch-add-update

# File: patch-memory.yaml
# -----------------------
# Patch limits.memory
apiVersion: apps/v1
kind: Deployment
# Set the desired deployment to patch
metadata:
  name: myapp
spec:
  # patch the memory limit
  template:
    spec:
      containers:
        - name: patch-name
          resources:
            limits:
              memory: 512Mi

# File: patch-replicas.yaml
# -------------------------
apiVersion: apps/v1
kind: Deployment
# Set the desired deployment to patch
metadata:
  name: myapp
spec:
  # This is the patch for this demo
  replicas: 3

# kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

bases:
  - ../_base

patchesStrategicMerge:
- patch-memory.yaml
- patch-replicas.yaml
  • Output:
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  # This is the first patch
  replicas: 3
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      # This is the second patch
      containers:
      - name: patch-name
        resources:
          limits:
            memory: 512Mi
      - image: __image__
        name: myapp

Patch-Delete

kubectl kustomize samples/08-Patches/patch-delete

# kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

bases:
  - ../../_base

patchesStrategicMerge:
- patch-delete.yaml

# patch-delete.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  template:
    spec:
      containers:
        # Remove this section, in this demo it will remove the 
        # image with the `name: myapp` 
        - $patch: delete
          name: myapp
          image: __image__
  • Output:
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - image: nginx
        name: nginx

Patch Replace

kubectl kustomize samples/08-Patches/patch-replace/

# kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

bases:
  - ../../_base

patchesStrategicMerge:
- patch-replace.yaml

# patch-replace.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  template:
    spec:
      containers:
        # Remove this section, in this demo it will remove the 
        # image with the `name: myapp` 
        - $patch: replace
        - name: myapp
          image: nginx:latest
          args:
          - one
          - two
  • Output:
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
spec:
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - args:
        - one
        - two
        image: nginx:latest
        name: myapp