Install and delete with helm
kubectl create clusterrolebinding cluster-admin-binding
--clusterrole cluster-admin
--user andy
helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts
helm install gatekeeper/gatekeeper --name-template=gatekeeper --namespace gatekeeper-system --create-namespace
helm delete gatekeeper --namespace gatekeeper-system
helm delete gatekeeper --namespace gatekeeper-system
Demos – Basic
❯ git clone git@github.com:open-policy-agent/gatekeeper.git
Cloning into 'gatekeeper'...
Enter passphrase for key '/Users/andy/.ssh/id_ed25519':
remote: Enumerating objects: 42115, done.
remote: Total 42115 (delta 0), reused 0 (delta 0), pack-reused 42115
Receiving objects: 100% (42115/42115), 98.19 MiB | 17.56 MiB/s, done.
Resolving deltas: 100% (23417/23417), done.
~/temp on ☁️ (us-west-2) took 16s
~/temp on ☁️ (us-west-2)
❯ cd gatekeeper
gatekeeper on master via 🐹 on ☁️ (us-west-2)
❯ cd demo
❯ ./demo.sh
$ kubectl apply -f sync.yaml
config.config.gatekeeper.sh/config created
$ kubectl create ns no-label
namespace/no-label created
$ cat templates/k8srequiredlabels_template.yaml
apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
name: k8srequiredlabels
spec:
crd:
spec:
names:
kind: K8sRequiredLabels
validation:
# Schema for the `parameters` field
openAPIV3Schema:
type: object
properties:
labels:
type: array
items:
type: string
targets:
- target: admission.k8s.gatekeeper.sh
rego: |
package k8srequiredlabels
violation[{"msg": msg, "details": {"missing_labels": missing}}] {
provided := {label | input.review.object.metadata.labels[label]}
required := {label | label := input.parameters.labels[_]}
missing := required - provided
count(missing) > 0
msg := sprintf("you must provide labels: %v", [missing])
}
$ kubectl apply -f templates/k8srequiredlabels_template.yaml
constrainttemplate.templates.gatekeeper.sh/k8srequiredlabels created
$ cat constraints/all_ns_must_have_gatekeeper.yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
name: ns-must-have-gk
spec:
match:
kinds:
- apiGroups: [""]
kinds: ["Namespace"]
parameters:
labels:
- key : "gatekeeper"
$ kubectl apply -f constraints/all_ns_must_have_gatekeeper.yaml
error: error validating "constraints/all_ns_must_have_gatekeeper.yaml": error validating data: ValidationError(K8sRequiredLabels.spec.parameters.labels[0]): invalid type for sh.gatekeeper.constraints.v1beta1.K8sRequiredLabels.spec.parameters.labels: got "map", expected "string"; if you choose to ignore these errors, turn validation off with --validate=false
$ kubectl apply -f bad/bad_ns.yaml
namespace/bad-ns created
$ cat good/good_ns.yaml
apiVersion: v1
kind: Namespace
metadata:
name: good-ns
labels:
"gatekeeper": "true"
$ kubectl apply -f good/good_ns.yaml
namespace/good-ns created
$ cat templates/k8suniquelabel_template.yaml
apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
name: k8suniquelabel
spec:
crd:
spec:
names:
kind: K8sUniqueLabel
validation:
# Schema for the `parameters` field
openAPIV3Schema:
type: object
properties:
label:
type: string
targets:
- target: admission.k8s.gatekeeper.sh
rego: |
package k8suniquelabel
make_apiversion(kind) = apiVersion {
g := kind.group
v := kind.version
g != ""
apiVersion = sprintf("%v/%v", [g, v])
}
make_apiversion(kind) = apiVersion {
kind.group == ""
apiVersion = kind.version
}
identical_namespace(obj, review) {
obj.metadata.namespace == review.namespace
obj.metadata.name == review.name
obj.kind == review.kind.kind
obj.apiVersion == make_apiversion(review.kind)
}
identical_cluster(obj, review) {
obj.metadata.name == review.name
obj.kind == review.kind.kind
obj.apiVersion == make_apiversion(review.kind)
}
violation[{"msg": msg, "details": {"value": val, "label": label}}] {
label := input.parameters.label
val := input.review.object.metadata.labels[label]
cluster_objs := [o | o = data.inventory.cluster[_][_][_]; not identical_cluster(o, input.review)]
ns_objs := [o | o = data.inventory.namespace[_][_][_][_]; not identical_namespace(o, input.review)]
all_objs := array.concat(cluster_objs, ns_objs)
all_values := {val | obj = all_objs[_]; val = obj.metadata.labels[label]}
count({val} - all_values) == 0
msg := sprintf("label %v has duplicate value %v", [label, val])
}
$ kubectl apply -f templates/k8suniquelabel_template.yaml
constrainttemplate.templates.gatekeeper.sh/k8suniquelabel created
$ kubectl apply -f constraints/all_ns_gatekeeper_label_unique.yaml
k8suniquelabel.constraints.gatekeeper.sh/ns-gk-label-unique created
$ cat good/no_dupe_ns.yaml
apiVersion: v1
kind: Namespace
metadata:
name: no-dupes
labels:
"gatekeeper": "not_duplicated"
$ kubectl apply -f good/no_dupe_ns.yaml
namespace/no-dupes created
$ cat bad/no_dupe_ns_2.yaml
apiVersion: v1
kind: Namespace
metadata:
name: no-dupes-2
labels:
"gatekeeper": "not_duplicated"
$ kubectl apply -f bad/no_dupe_ns_2.yaml
Error from server (Forbidden): error when creating "bad/no_dupe_ns_2.yaml": admission webhook "validation.gatekeeper.sh" denied the request: [ns-gk-label-unique] label gatekeeper has duplicate value not_duplicated
$ kubectl get k8srequiredlabels ns-must-have-gk -o yaml
Error from server (NotFound): k8srequiredlabels.constraints.gatekeeper.sh "ns-must-have-gk" not found
$ THE END
k8suniquelabel.constraints.gatekeeper.sh "ns-gk-label-unique" deleted
Error from server (NotFound): error when deleting "constraints/all_ns_must_have_gatekeeper.yaml": k8srequiredlabels.constraints.gatekeeper.sh "ns-must-have-gk" not found
Error from server (NotFound): error when deleting "constraints/all_ns_must_have_gatekeeper_dryrun.yaml": k8srequiredlabels.constraints.gatekeeper.sh "ns-must-have-gk-dryrun" not found
constrainttemplate.templates.gatekeeper.sh "k8srequiredlabels" deleted
constrainttemplate.templates.gatekeeper.sh "k8suniquelabel" deleted
Error from server (NotFound): error when deleting "templates/k8srequiredlabels_template_external_data.yaml": constrainttemplates.templates.gatekeeper.sh "k8srequiredlabels" not found
namespace "good-ns" deleted
namespace "no-dupes" deleted
namespace "no-label" deleted
config.config.gatekeeper.sh "config" deleted
gatekeeper/demo/basic on master on ☁️ (us-west-2) took 27m1s
❯ cd ..
gatekeeper/demo on master on ☁️ (us-west-2)
❯ cd agilebank
Demos – agilebank
gatekeeper/demo/basic on master on ☁️ (us-west-2) took 27m1s
❯ cd ..
gatekeeper/demo on master on ☁️ (us-west-2)
❯ cd agilebank
gatekeeper/demo/agilebank on master on ☁️ (us-west-2)
❯ ./demo.sh
Error from server (BadRequest): error when creating "dryrun/existing_resources/example1.yaml": admission webhook "validate.nginx.ingress.kubernetes.io" denied the request: host "example-host.example.com" and path "/" is already defined in ingress default/ingress-host
===== ENTER developer =====
$ kubectl create ns advanced-transaction-system
namespace/advanced-transaction-system created
===== EXIT developer =====
$ Five weeks go by. Developer moves on to another project. "advanced-transaction-system" is long forgotten until...
$ Our intrepid admin finds it. "What is advanced-transaction-system? Do we use it?"
$ They go on a three day quest across many departments...
$ Only to find the project was scrapped.
$ "Never again," says the admin as they delete the namespace.
===== ENTER admin =====
$ ls -1 templates
k8sallowedrepos_template.yaml
k8scontainterlimits_template.yaml
k8srequiredlabels_template.yaml
k8srequiredprobes_template.yaml
k8suniqueserviceselector_template.yaml
$ kubectl apply -f templates
constrainttemplate.templates.gatekeeper.sh/k8sallowedrepos created
constrainttemplate.templates.gatekeeper.sh/k8scontainerlimits created
constrainttemplate.templates.gatekeeper.sh/k8srequiredlabels created
constrainttemplate.templates.gatekeeper.sh/k8srequiredprobes created
constrainttemplate.templates.gatekeeper.sh/k8suniqueserviceselector created
$ ls -1 constraints
containers_must_be_limited.yaml
owner_must_be_provided.yaml
probes_must_be_provided.yaml
prod_repo_is_openpolicyagent.yaml
unique_service_selector.yaml
$ cat constraints/owner_must_be_provided.yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
name: all-must-have-owner
spec:
match:
kinds:
- apiGroups: [""]
kinds: ["Namespace"]
parameters:
message: "All namespaces must have an `owner` label that points to your company username"
labels:
- key: owner
allowedRegex: "^[a-zA-Z]+.agilebank.demo$"
$ kubectl apply -f constraints
k8scontainerlimits.constraints.gatekeeper.sh/container-must-have-limits created
k8srequiredlabels.constraints.gatekeeper.sh/all-must-have-owner created
k8srequiredprobes.constraints.gatekeeper.sh/must-have-probes created
k8sallowedrepos.constraints.gatekeeper.sh/prod-repo-is-openpolicyagent created
k8suniqueserviceselector.constraints.gatekeeper.sh/unique-service-selector created
===== ENTER developer ======
$ kubectl create ns production
Error from server (Forbidden): admission webhook "validation.gatekeeper.sh" denied the request: [all-must-have-owner] All namespaces must have an `owner` label that points to your company username
$ cat good_resources/namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: production
labels:
owner: "me.agilebank.demo"
$ kubectl apply -f good_resources/namespace.yaml
namespace/production created
$ kubectl apply -f bad_resources/opa_no_limits.yaml
Error from server (Forbidden): error when creating "bad_resources/opa_no_limits.yaml": admission webhook "validation.gatekeeper.sh" denied the request: [must-have-probes] Container <opa> in your <Pod> <opa> has no <livenessProbe>
[must-have-probes] Container <opa> in your <Pod> <opa> has no <readinessProbe>
[container-must-have-limits] container <opa> has no resource limits
$ kubectl apply -f bad_resources/opa_limits_too_high.yaml
Error from server (Forbidden): error when creating "bad_resources/opa_limits_too_high.yaml": admission webhook "validation.gatekeeper.sh" denied the request: [must-have-probes] Container <opa> in your <Pod> <opa> has no <livenessProbe>
[must-have-probes] Container <opa> in your <Pod> <opa> has no <readinessProbe>
[container-must-have-limits] container <opa> cpu limit <300m> is higher than the maximum allowed of <200m>
[container-must-have-limits] container <opa> memory limit <4000Mi> is higher than the maximum allowed of <1Gi>
$ kubectl apply -f bad_resources/opa_wrong_repo.yaml
Error from server (Forbidden): error when creating "bad_resources/opa_wrong_repo.yaml": admission webhook "validation.gatekeeper.sh" denied the request: [must-have-probes] Container <opa> in your <Pod> <opa> has no <livenessProbe>
[must-have-probes] Container <opa> in your <Pod> <opa> has no <readinessProbe>
[prod-repo-is-openpolicyagent] container <opa> has an invalid image repo <gcr.io/smythe-kpc/testbuilds/opa:0.9.2>, allowed repos are ["openpolicyagent"]
$ kubectl apply -f good_resources/opa.yaml
Error from server (Forbidden): error when creating "good_resources/opa.yaml": admission webhook "validation.gatekeeper.sh" denied the request: [must-have-probes] Container <opa> in your <Pod> <opa> has no <livenessProbe>
[must-have-probes] Container <opa> in your <Pod> <opa> has no <readinessProbe>
$ cat bad_resources/duplicate_service.yaml
apiVersion: v1
kind: Service
metadata:
name: gatekeeper-test-service
namespace: production
spec:
ports:
- port: 443
selector:
control-plane: controller-manager
$ kubectl apply -f bad_resources/duplicate_service.yaml
service/gatekeeper-test-service created
$ After some more trial and error, the developer's service is up and running
===== EXIT developer =====
$ All is well with the world, until the big outage. The bank is down for hours.
===== ENTER admin =====
$ We had no idea there were resources in the cluster without resource limits. Now they are causing issues in production!
$ We need to get all the resources in the cluster that lack resource limits.
$ Let's check out the audit results of the container-must-have-limits constraint!
$ kubectl get k8scontainerlimits.constraints.gatekeeper.sh container-must-have-limits -o yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sContainerLimits
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"constraints.gatekeeper.sh/v1beta1","kind":"K8sContainerLimits","metadata":{"annotations":{},"name":"container-must-have-limits"},"spec":{"match":{"kinds":[{"apiGroups":[""],"kinds":["Pod"]}]},"parameters":{"cpu":"200m","memory":"1Gi"}}}
creationTimestamp: "2022-04-23T16:09:42Z"
generation: 1
name: container-must-have-limits
resourceVersion: "14061"
uid: f66b8345-44a8-4208-a9d5-99385f9a91e9
spec:
match:
kinds:
- apiGroups:
- ""
kinds:
- Pod
parameters:
cpu: 200m
memory: 1Gi
status:
auditTimestamp: "2022-04-23T17:16:16Z"
byPod:
- constraintUID: f66b8345-44a8-4208-a9d5-99385f9a91e9
enforced: true
id: gatekeeper-audit-68d875d865-hnpbs
observedGeneration: 1
operations:
- audit
- mutation-status
- status
- constraintUID: f66b8345-44a8-4208-a9d5-99385f9a91e9
enforced: true
id: gatekeeper-controller-manager-7c6d5d97fc-bjs6b
observedGeneration: 1
operations:
- mutation-webhook
- webhook
- constraintUID: f66b8345-44a8-4208-a9d5-99385f9a91e9
enforced: true
id: gatekeeper-controller-manager-7c6d5d97fc-p627q
observedGeneration: 1
operations:
- mutation-webhook
- webhook
- constraintUID: f66b8345-44a8-4208-a9d5-99385f9a91e9
enforced: true
id: gatekeeper-controller-manager-7c6d5d97fc-wj5xq
observedGeneration: 1
operations:
- mutation-webhook
- webhook
totalViolations: 14
violations:
- enforcementAction: deny
kind: Pod
message: container <etcd> has no resource limits
name: etcd-minikube
namespace: kube-system
- enforcementAction: deny
kind: Pod
message: container <kube-apiserver> has no resource limits
name: kube-apiserver-minikube
namespace: kube-system
- enforcementAction: deny
kind: Pod
message: container <manager> cpu limit <1> is higher than the maximum allowed
of <200m>
name: gatekeeper-controller-manager-7c6d5d97fc-wj5xq
namespace: gatekeeper-system
- enforcementAction: deny
kind: Pod
message: container <storage-provisioner> has no resource limits
name: storage-provisioner
namespace: kube-system
- enforcementAction: deny
kind: Pod
message: container <kube-controller-manager> has no resource limits
name: kube-controller-manager-minikube
namespace: kube-system
- enforcementAction: deny
kind: Pod
message: container <kube-scheduler> has no resource limits
name: kube-scheduler-minikube
namespace: kube-system
- enforcementAction: deny
kind: Pod
message: container <manager> cpu limit <1> is higher than the maximum allowed
of <200m>
name: gatekeeper-controller-manager-7c6d5d97fc-bjs6b
namespace: gatekeeper-system
- enforcementAction: deny
kind: Pod
message: container <manager> cpu limit <1> is higher than the maximum allowed
of <200m>
name: gatekeeper-controller-manager-7c6d5d97fc-p627q
namespace: gatekeeper-system
- enforcementAction: deny
kind: Pod
message: container <kube-proxy> has no resource limits
name: kube-proxy-6sdw5
namespace: kube-system
- enforcementAction: deny
kind: Pod
message: container <manager> cpu limit <1> is higher than the maximum allowed
of <200m>
name: gatekeeper-audit-68d875d865-hnpbs
namespace: gatekeeper-system
- enforcementAction: deny
kind: Pod
message: container <create> has no resource limits
name: ingress-nginx-admission-create-x2vjj
namespace: ingress-nginx
- enforcementAction: deny
kind: Pod
message: container <coredns> has no cpu limit
name: coredns-64897985d-xgz52
namespace: kube-system
- enforcementAction: deny
kind: Pod
message: container <controller> has no resource limits
name: ingress-nginx-controller-cc8496874-2b5xw
namespace: ingress-nginx
- enforcementAction: deny
kind: Pod
message: container <patch> has no resource limits
name: ingress-nginx-admission-patch-xfk4j
namespace: ingress-nginx
$ Weeks gone by, the company now has a new policy to rollout to production.
$ We need to ensure all ingress hostnames are unique.
$ Introducing new policies is dangerous and can often be a breaking change. How do we gain confidence in the new policy before enforcing them in production? What if it breaks a core piece of software, what if it brings down the entire stack?!
===== ENTER admin =====
$ We can use the dry run feature in Gatekeeper to preview the effect of policy changes in production without impacting the workload and our users!
$ kubectl apply -f dryrun/k8suniqueingresshost_template.yaml
constrainttemplate.templates.gatekeeper.sh/k8suniqueingresshost created
$ kubectl apply -f dryrun/unique-ingress-host.yaml
k8suniqueingresshost.constraints.gatekeeper.sh/unique-ingress-host created
$ cat dryrun/k8suniqueingresshost_template.yaml
apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
name: k8suniqueingresshost
spec:
crd:
spec:
names:
kind: K8sUniqueIngressHost
targets:
- target: admission.k8s.gatekeeper.sh
rego: |
package k8suniqueingresshost
identical(obj, review) {
obj.metadata.namespace == review.object.metadata.namespace
obj.metadata.name == review.object.metadata.name
}
violation[{"msg": msg}] {
input.review.kind.kind == "Ingress"
re_match("^(extensions|networking.k8s.io)$", input.review.kind.group)
host := input.review.object.spec.rules[_].host
other := data.inventory.namespace[ns][otherapiversion]["Ingress"][name]
re_match("^(extensions|networking.k8s.io)/.+$", otherapiversion)
other.spec.rules[_].host == host
not identical(other, input.review)
msg := sprintf("ingress host conflicts with an existing ingress <%v>", [host])
}
$ cat dryrun/unique-ingress-host.yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sUniqueIngressHost
metadata:
name: unique-ingress-host
spec:
enforcementAction: dryrun
match:
kinds:
- apiGroups: ["extensions", "networking.k8s.io"]
kinds: ["Ingress"]
$ kubectl get ingress --all-namespaces
NAMESPACE NAME CLASS HOSTS ADDRESS PORTS AGE
default ingress-host nginx example-host.example.com 192.168.49.2 80 3h23m
$ kubectl get K8sUniqueIngressHost.constraints.gatekeeper.sh unique-ingress-host -o yaml
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sUniqueIngressHost
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"constraints.gatekeeper.sh/v1beta1","kind":"K8sUniqueIngressHost","metadata":{"annotations":{},"name":"unique-ingress-host"},"spec":{"enforcementAction":"dryrun","match":{"kinds":[{"apiGroups":["extensions","networking.k8s.io"],"kinds":["Ingress"]}]}}}
creationTimestamp: "2022-04-23T19:06:41Z"
generation: 1
name: unique-ingress-host
resourceVersion: "18473"
uid: 5adc03ff-3cd0-40b4-b655-53c42909583e
spec:
enforcementAction: dryrun
match:
kinds:
- apiGroups:
- extensions
- networking.k8s.io
kinds:
- Ingress
status:
auditTimestamp: "2022-04-23T19:06:56Z"
byPod:
- constraintUID: 5adc03ff-3cd0-40b4-b655-53c42909583e
enforced: true
id: gatekeeper-audit-68d875d865-hnpbs
observedGeneration: 1
operations:
- audit
- mutation-status
- status
- constraintUID: 5adc03ff-3cd0-40b4-b655-53c42909583e
enforced: true
id: gatekeeper-controller-manager-7c6d5d97fc-bjs6b
observedGeneration: 1
operations:
- mutation-webhook
- webhook
- constraintUID: 5adc03ff-3cd0-40b4-b655-53c42909583e
enforced: true
id: gatekeeper-controller-manager-7c6d5d97fc-p627q
observedGeneration: 1
operations:
- mutation-webhook
- webhook
- constraintUID: 5adc03ff-3cd0-40b4-b655-53c42909583e
enforced: true
id: gatekeeper-controller-manager-7c6d5d97fc-wj5xq
observedGeneration: 1
operations:
- mutation-webhook
- webhook
totalViolations: 0
$ Let's see if this new policy in dry run mode blocks operations on the cluster?
$ cat dryrun/bad_resource/duplicate_ing.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-host2
spec:
rules:
- host: example-host.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: nginx3
port:
number: 80
$ kubectl apply -f dryrun/bad_resource/duplicate_ing.yaml
Error from server (BadRequest): error when creating "dryrun/bad_resource/duplicate_ing.yaml": admission webhook "validate.nginx.ingress.kubernetes.io" denied the request: host "example-host.example.com" and path "/" is already defined in ingress default/ingress-host
$ THE END
ingress.networking.k8s.io "ingress-host" deleted
Error from server (NotFound): error when deleting "dryrun/existing_resources/example1.yaml": ingresses.networking.k8s.io "ingress-host1" not found
Error from server (NotFound): error when deleting "dryrun/bad_resource/duplicate_ing.yaml": ingresses.networking.k8s.io "ingress-host2" not found
constrainttemplate.templates.gatekeeper.sh "k8suniqueingresshost" deleted
k8suniqueingresshost.constraints.gatekeeper.sh "unique-ingress-host" deleted
namespace "production" deleted
Error from server (NotFound): error when deleting "good_resources/opa.yaml": pods "opa" not found
namespace "advanced-transaction-system" deleted
k8scontainerlimits.constraints.gatekeeper.sh "container-must-have-limits" deleted
k8srequiredlabels.constraints.gatekeeper.sh "all-must-have-owner" deleted
k8srequiredprobes.constraints.gatekeeper.sh "must-have-probes" deleted
k8sallowedrepos.constraints.gatekeeper.sh "prod-repo-is-openpolicyagent" deleted
k8suniqueserviceselector.constraints.gatekeeper.sh "unique-service-selector" deleted
constrainttemplate.templates.gatekeeper.sh "k8sallowedrepos" deleted
constrainttemplate.templates.gatekeeper.sh "k8scontainerlimits" deleted
constrainttemplate.templates.gatekeeper.sh "k8srequiredlabels" deleted
constrainttemplate.templates.gatekeeper.sh "k8srequiredprobes" deleted
constrainttemplate.templates.gatekeeper.sh "k8suniqueserviceselector" deleted
config.config.gatekeeper.sh "config" deleted
gatekeeper/demo/agilebank on master on ☁️ (us-west-2) took 3h24m21s