Kubernetes：多叢集管理策略

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: default
  name: pod-viewer
rules:
- apiGroups: [""] # "" 表示核心 API 組
  resources: ["pods"]
  verbs: ["get", "watch", "list"]

kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: noc-helpdesk-view
  namespace: default
subjects:
- kind: User
  name: helpdeskuser@example.com
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role #這必須是 Role 或 ClusterRole
  name: pod-viewer # 這必須比對要繫結的 Role 或 ClusterRole 的名稱
  apiGroup: rbac.authorization.k8s.io

regions:
  - name: asia-east1
    location: "台灣"
    purpose: "服務亞洲東部使用者"
  - name: europe-west3
    location: "德國"
    purpose: "服務歐洲使用者"
  - name: us-central1
    location: "美國中部"
    purpose: "服務北美使用者與備援"

apiVersion: v1
kind: Pod
metadata:
  name: resource-demo
spec:
  containers:
  - name: resource-demo-ctr
    image: nginx
    resources:
      requests:
        memory: "64Mi"
        cpu: "250m"
      limits:
        memory: "128Mi"
        cpu: "500m"

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: api-allow
spec:
  podSelector:
    matchLabels:
      app: api-service
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend
    ports:
    - protocol: TCP
      port: 8080
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: database
    ports:
    - protocol: TCP
      port: 5432

apiVersion: v1
kind: Pod
metadata:
  name: security-enhanced-pod
spec:
  securityContext:
    runAsNonRoot: true
    fsGroup: 2000
  containers:
  - name: secure-app
    image: my-secure-app:1.0.2
    securityContext:
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: true
      capabilities:
        drop:
        - ALL
        add:
        - NET_BIND_SERVICE

apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
  name: require-team-label
spec:
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Namespace"]
  parameters:
    labels: ["team", "environment"]
---
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8srequiredlabels
spec:
  crd:
    spec:
      names:
        kind: K8sRequiredLabels
      validation:
        openAPIV3Schema:
          properties:
            labels:
              type: array
              items:
                type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8srequiredlabels
        
        violation[{"msg": msg}] {
          provided := {label | input.review.object.metadata.labels[label]}
          required := {label | label := input.parameters.labels[_]}
          missing := required - provided
          count(missing) > 0
          msg := sprintf("missing required labels: %v", [missing])
        }

kind: GitRepo
apiVersion: fleet.cattle.io/v1alpha1
metadata:
  name: global-config
  namespace: fleet-default
spec:
  repo: https://github.com/organization/global-configs
  branch: main
  paths:
  - /base
  targets:
  - name: production
    clusterSelector:
      matchLabels:
        environment: production
  - name: development
    clusterSelector:
      matchLabels:
        environment: development
    overlays:
    - /overlays/development

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: deny-specific-requests
webhooks:
- name: deny.example.com
  clientConfig:
    service:
      namespace: admission-webhook
      name: admission-webhook-service
      path: "/validate"
    caBundle: <base64-encoded-ca-cert>
  rules:
  - apiGroups: ["apps"]
    apiVersions: ["v1"]
    operations: ["CREATE", "UPDATE"]
    resources: ["deployments"]
    scope: "Namespaced"
  admissionReviewVersions: ["v1"]
  sideEffects: None
  timeoutSeconds: 5
  failurePolicy: Fail

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "net/http"
    
    admissionv1 "k8s.io/api/admission/v1"
    appsv1 "k8s.io/api/apps/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/klog"
)

// 處理驗證請求
func validateDeployments(w http.ResponseWriter, r *http.Request) {
    var body []byte
    if r.Body != nil {
        if data, err := ioutil.ReadAll(r.Body); err == nil {
            body = data
        }
    }
    
    // 解析 AdmissionReview 請求
    requestedAdmissionReview := admissionv1.AdmissionReview{}
    if err := json.Unmarshal(body, &requestedAdmissionReview); err != nil {
        klog.Error(err)
        w.WriteHeader(http.StatusBadRequest)
        return
    }
    
    // 提取佈署資源
    var deployment appsv1.Deployment
    err := json.Unmarshal(requestedAdmissionReview.Request.Object.Raw, &deployment)
    if err != nil {
        klog.Error(err)
        w.WriteHeader(http.StatusBadRequest)
        return
    }
    
    // 檢查是否符合規則
    allowed := true
    var result *metav1.Status
    
    // 範例規則：拒絕沒有設定資源限制的佈署
    if deployment.Spec.Template.Spec.Containers[0].Resources.Limits == nil {
        allowed = false
        result = &metav1.Status{
            Reason: "必須設定資源限制",
        }
    }
    
    // 建立回應
    responseAdmissionReview := admissionv1.AdmissionReview{
        TypeMeta: requestedAdmissionReview.TypeMeta,
        Response: &admissionv1.AdmissionResponse{
            UID:     requestedAdmissionReview.Request.UID,
            Allowed: allowed,
            Result:  result,
        },
    }
    
    // 回傳 JSON 回應
    resp, err := json.Marshal(responseAdmissionReview)
    if err != nil {
        klog.Error(err)
        w.WriteHeader(http.StatusInternalServerError)
        return
    }
    
    w.Header().Set("Content-Type", "application/json")
    w.Write(resp)
}

func main() {
    http.HandleFunc("/validate", validateDeployments)
    klog.Info("准入控制器服務啟動於 :8443")
    klog.Fatal(http.ListenAndServeTLS(":8443", "/certs/tls.crt", "/certs/tls.key", nil))
}

apiVersion: v1
kind: Namespace
metadata:
  name: admission-webhook
apiVersion: v1
kind: Service
metadata:
  name: admission-webhook-service
  namespace: admission-webhook
spec:
  selector:
    app: admission-webhook
  ports:
  - port: 443
    targetPort: 8443

#!/bin/bash

# 設定變數
SERVICE=admission-webhook-service
NAMESPACE=admission-webhook
SECRET_NAME=webhook-certs

# 建立臨時目錄
mkdir -p certs
cd certs

# 生成 CA 私鑰和證書
openssl genrsa -out ca.key 2048
openssl req -new -x509 -key ca.key -out ca.crt -subj "/CN=admission-webhook-ca"

# 生成伺服器私鑰
openssl genrsa -out server.key 2048

# 生成伺服器證書籤名請求
cat > server.conf << EOF
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]

[v3_req]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1 = ${SERVICE}
DNS.2 = ${SERVICE}.${NAMESPACE}
DNS.3 = ${SERVICE}.${NAMESPACE}.svc
EOF

openssl req -new -key server.key -out server.csr -subj "/CN=${SERVICE}.${NAMESPACE}.svc" -config server.conf

# 簽署伺服器證書
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -extensions v3_req -extfile server.conf

# 建立 Kubernetes Secret
kubectl create secret generic ${SECRET_NAME} \
    --from-file=tls.key=server.key \
    --from-file=tls.crt=server.crt \
    --dry-run=client -o yaml | kubectl -n ${NAMESPACE} apply -f -

# 取得 CA 證書的 base64 編碼
CA_BUNDLE=$(cat ca.crt | base64 | tr -d '\n')

# 輸出 CA 證書的 base64 編碼
echo "CA Bundle for webhook configuration: ${CA_BUNDLE}"

# 清理
cd ..

apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: pod-label-mutator
webhooks:
- name: mutate.pods.example.com
  clientConfig:
    service:
      namespace: admission-webhook
      name: admission-webhook-service
      path: "/mutate"
    caBundle: <base64-encoded-ca-cert>
  rules:
  - apiGroups: [""]
    apiVersions: ["v1"]
    operations: ["CREATE"]
    resources: ["pods"]
    scope: "Namespaced"
  admissionReviewVersions: ["v1"]
  sideEffects: None
  timeoutSeconds: 5
  failurePolicy: Ignore

func mutatePods(w http.ResponseWriter, r *http.Request) {
    var body []byte
    if r.Body != nil {
        if data, err := ioutil.ReadAll(r.Body); err == nil {
            body = data
        }
    }
    
    // 解析 AdmissionReview 請求
    requestedAdmissionReview := admissionv1.AdmissionReview{}
    if err := json.Unmarshal(body, &requestedAdmissionReview); err != nil {
        klog.Error(err)
        w.WriteHeader(http.StatusBadRequest)
        return
    }
    
    // 提取 Pod 資源
    var pod corev1.Pod
    err := json.Unmarshal(requestedAdmissionReview.Request.Object.Raw, &pod)
    if err != nil {
        klog.Error(err)
        w.WriteHeader(http.StatusBadRequest)
        return
    }
    
    // 建立修改操作的 JSON patch
    var patches []map[string]string
    
    // 如果 Pod 沒有標籤，先建立標籤欄位
    if pod.Labels == nil {
        patches = append(patches, map[string]string{
            "op":    "add",
            "path":  "/metadata/labels",
            "value": "{}",
        })
    }
    
    // 增加自訂標籤
    patches = append(patches, map[string]string{
        "op":    "add",
        "path":  "/metadata/labels/injected-by",
        "value": "admission-webhook",
    })
    
    // 序列化 patch
    patchBytes, err := json.Marshal(patches)
    if err != nil {
        klog.Error(err)
        w.WriteHeader(http.StatusInternalServerError)
        return
    }
    
    // 建立回應
    responseAdmissionReview := admissionv1.AdmissionReview{
        TypeMeta: requestedAdmissionReview.TypeMeta,
        Response: &admissionv1.AdmissionResponse{
            UID:     requestedAdmissionReview.Request.UID,
            Allowed: true,
            Patch:   patchBytes,
            PatchType: func() *admissionv1.PatchType {
                pt := admissionv1.PatchTypeJSONPatch
                return &pt
            }(),
        },
    }
    
    // 回傳 JSON 回應
    resp, err := json.Marshal(responseAdmissionReview)
    if err != nil {
        klog.Error(err)
        w.WriteHeader(http.StatusInternalServerError)
        return
    }
    
    w.Header().Set("Content-Type", "application/json")
    w.Write(resp)
}

// 檢查容器是否設定了資源請求和限制
func validateResourceRequirements(container corev1.Container) (bool, string) {
    if container.Resources.Requests == nil || container.Resources.Limits == nil {
        return false, "所有容器必須設定資源請求和限制"
    }
    
    cpuRequest := container.Resources.Requests.Cpu()
    memoryRequest := container.Resources.Requests.Memory()
    cpuLimit := container.Resources.Limits.Cpu()
    memoryLimit := container.Resources.Limits.Memory()
    
    if cpuRequest.IsZero() || memoryRequest.IsZero() {
        return false, "CPU 和記憶體請求不能為零"
    }
    
    if cpuLimit.IsZero() || memoryLimit.IsZero() {
        return false, "CPU 和記憶體限制不能為零"
    }
    
    // 確保限制大於請求
    if cpuLimit.Cmp(*cpuRequest) < 0 {
        return false, "CPU 限制必須大於或等於請求"
    }
    
    if memoryLimit.Cmp(*memoryRequest) < 0 {
        return false, "記憶體限制必須大於或等於請求"
    }
    
    return true, ""
}

// 檢查容器映像是否來自允許的倉函式庫
func validateImageSource(container corev1.Container) (bool, string) {
    allowedRegistries := []string{
        "registry.company.com/",
        "gcr.io/company-project/",
        "docker.io/company/",
    }
    
    for _, registry := range allowedRegistries {
        if strings.HasPrefix(container.Image, registry) {
            return true, ""
        }
    }
    
    return false, fmt.Sprintf("容器映像必須來自允許的倉函式庫: %v", allowedRegistries)
}

// 檢查名稱空間是否有網路策略
func validateNetworkPolicy(namespace string, client kubernetes.Interface) (bool, string) {
    policies, err := client.NetworkingV1().NetworkPolicies(namespace).List(context.TODO(), metav1.ListOptions{})
    if err != nil {
        return false, fmt.Sprintf("無法檢查網路策略: %v", err)
    }
    
    if len(policies.Items) == 0 {
        return false, "名稱空間必須至少有一個網路策略"
    }
    
    return true, ""
}

func validateSecurityContext(pod corev1.Pod) (bool, string) {
    // 檢查 Pod 安全上下文
    if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.RunAsNonRoot != nil && *pod.Spec.SecurityContext.RunAsNonRoot == false {
        return false, "Pod 必須以非 root 使用者執行"
    }
    
    // 檢查每個容器的安全上下文
    for _, container := range pod.Spec.Containers {
        if container.SecurityContext != nil {
            // 檢查特權模式
            if container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged {
                return false, "不允許特權容器"
            }
            
            // 檢查 capabilities
            if container.SecurityContext.Capabilities != nil && len(container.SecurityContext.Capabilities.Add) > 0 {
                for _, cap := range container.SecurityContext.Capabilities.Add {
                    if cap == "SYS_ADMIN" || cap == "NET_ADMIN" {
                        return false, fmt.Sprintf("不允許增加敏感 capability: %s", cap)
                    }
                }
            }
            
            // 檢查是否以 root 執行
            if container.SecurityContext.RunAsNonRoot != nil && *container.SecurityContext.RunAsNonRoot == false {
                return false, "容器必須以非 root 使用者執行"
            }
        }
    }
    
    return true, ""
}

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: opa-validating-webhook
webhooks:
- name: validating.opa.example.com
  clientConfig:
    service:
      namespace: opa
      name: opa
      path: "/v1/data/kubernetes/admission"
    caBundle: <base64-encoded-ca-cert>
  rules:
  - apiGroups: ["*"]
    apiVersions: ["*"]
    operations: ["CREATE", "UPDATE"]
    resources: ["*"]
    scope: "*"
  admissionReviewVersions: ["v1"]
  sideEffects: None
  timeoutSeconds: 5
  failurePolicy: Fail

package kubernetes.admission

deny[msg] {
    input.request.kind.kind == "Pod"
    container := input.request.object.spec.containers[_]
    container.securityContext.privileged
    msg := sprintf("特權容器不允許: %v", [container.name])
}

deny[msg] {
    input.request.kind.kind == "Pod"
    not input.request.object.metadata.labels.app
    msg := "Pod 必須有 app 標籤"
}

kubectl get validatingwebhookconfiguration -o yaml
kubectl get mutatingwebhookconfiguration -o yaml