kubernetes ingress with TLS

  • Create self signed cert
openssl req \
    -new \
    -newkey rsa:4096 \
    -days 365 \
    -nodes \
    -x509 \
    -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=example.com" \
    -keyout example.com.key \
    -out example.com.cert
  • Create k8 certificate using above cert
kubectl create secret tls example-cert \
  --key="example.com.key" \
  --cert="example.com.cert"
  • ingress.yaml file
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: "haproxy"
    haproxy.org/rewrite-target: "/"
  name: prometheus-ingress
spec:
  rules:
  - host: prometheus.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: prometheus-service
            port:
              number: 9090
 tls:
 - secretName: example-cert
   hosts:
   - prometheus.example.com

Custom Daemonset command based on host_ip in kubernetes

Why?
– When we need to add some extra functionally to daemonset based on which worker node it’s running on

apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: custom-daemonset
  name: custom-daemonset
spec:
  selector:
    matchLabels:
      app: custom-daemonset
  template:
    metadata:
      labels:
        app: custom-daemonset
    spec:
      containers:
      - command:
        - /bin/bash
        - -c
        - |
          echo "$STARTUP_SCRIPT" > /tmp/STARTUP_SCRIPT.sh
          /bin/bash /tmp/STARTUP_SCRIPT.sh
        env:
        - name: HOST_IP
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: STARTUP_SCRIPT
          value: |
            #!/bin/bash
            if [ $HOST_IP == "192.168.0.184" ]; then
              echo "HOST_IP is $HOST_IP"
            else
              echo "HOST_IP does not match $HOST_IP"
            fi
            sleep 600
        image: nginx
        imagePullPolicy: IfNotPresent
        name: custom-daemonset

Ref : https://github.com/kubernetes/kubernetes/issues/24657#issuecomment-577747926

simple redis deployment in kubernetes

apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-deployment
  labels:
    app: redis
    env: prod
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis
      env: prod
  template:
    metadata:
      labels:
        app: redis
        env: prod
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: "kubernetes.io/arch"
                operator: "In"
                values:
                - arm64
      tolerations:
      - key: "key"
        operator: "Equal"
        value: "arm"
        effect: "NoSchedule" 
      containers:
      - name: redis-container
        image: redis:6.2
        imagePullPolicy: IfNotPresent
        resources:
          requests:
            memory: "128Mi"
            cpu: "100m"
          limits:
            memory: "512Mi"
            cpu: "300m"
        volumeMounts:
          - name: redis-data
            mountPath: /data
        ports:
        - containerPort: 9090
      volumes:
        - name: redis-data
          nfs:
            server: 192.168.0.182
            path: "/mnt/nfs1/redis"
---
kind: Service
apiVersion: v1
metadata:
  name: redis-service
  labels:
    app: redis
    env: prod
spec:
  selector:
    app: redis
    env: prod
  ports:
  - name: redis
    protocol: TCP
    port: 6379
    targetPort: 6379
    nodePort: 31000
  type: NodePort

redis cli:

127.0.0.1:6379> set var1 100
OK
127.0.0.1:6379> get var1
"100"
127.0.0.1:6379> incr var1
(integer) 101
127.0.0.1:6379> get var1
"101"

raspberry pi as kubernetes worker node

  • Install docker
apt-get install \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg \
    lsb-release

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

echo \
  "deb [arch=arm64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

apt-get update

apt-get install docker-ce docker-ce-cli containerd.io

https://docs.docker.com/engine/install/ubuntu/

echo 'cgroup_memory=1' > /boot/cmdline.txt


echo '{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
' > /etc/docker/daemon.json



sudo sed -i '$ s/$/ cgroup_enable=cpuset cgroup_enable=memory cgroup_memory=1 swapaccount=1/' /boot/firmware/cmdline.txt


cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sudo sysctl --system




  • Install kubernetes component
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -

cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main
EOF

apt update && sudo apt install -y kubelet kubeadm kubectl


echo '[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cgroup-driver=systemd"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS' > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf


systemctl daemon-reload

systemctl restart kubelet

https://opensource.com/article/20/6/kubernetes-raspberry-pi

https://stackoverflow.com/questions/45708175/kubelet-failed-with-kubelet-cgroup-driver-cgroupfs-is-different-from-docker-c

Delete k8 false apiservice – namespace

kubectl api-resources 
# look for which apiservice is giving error

kubectl get apiservice
# look for which False and MissingEndpoints

kubectl delete apiservice <service-name>

kubectl api-resources

# get CRD related to api-resources
kubectl get crds | grep cilium

kubectl delete crd ciliumnodes.cilium.io

https://github.com/helm/helm/issues/6361#issuecomment-538220109

  • delete namcespace
NAMESPACE=your_namespace
kubectl proxy &
kubectl get namespace $NAMESPACE -o json |jq '.spec = {"finalizers":[]}' >temp.json
curl -k -H "Content-Type: application/json" -X PUT --data-binary @temp.json 127.0.0.1:8001/api/v1/namespaces/$NAMESPACE/finalize

https://github.com/helm/helm/issues/6361

  • Delete rook-ceph namespace
kubectl -n rook-ceph patch cephclusters.ceph.rook.io rook-ceph -p '{"metadata":{"finalizers": []}}' --type=merge

kubectl api-resources --verbs=list --namespaced -o name  | xargs -n 1 kubectl get --show-kind --ignore-not-found -n rook-ceph

More : https://github.com/rook/rook/issues/2668

Install metric server in kubernetes

WHY?
– Get node CPU/RAM usages
– Can create Horizontal Pod Autoscaler (HPA)
– Light weight

git clone https://github.com/kubernetes-sigs/metrics-server.git
  • Edit metrics-server/manifests/base/deployment.yaml and add below lines to args
args:
          - --cert-dir=/tmp
          - --secure-port=4443
          - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP
          - --kubelet-use-node-status-port #Deprecated metrics-server:v0.3.7
          - --kubelet-insecure-tls
kubectl apply -f metrics-server/manifests/base
  • To get node metrics run kubectl get top node

prometheus blackbox exporter in Kubernetes

prometheus-blackbox.yml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-blackbox-exporter
  labels:
    app: prometheus-blackbox-exporter
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus-blackbox-exporter
  template:
    metadata:
      labels:
        app: prometheus-blackbox-exporter
    spec:
      restartPolicy: Always
      containers:
        - name: blackbox-exporter
          image: "prom/blackbox-exporter:v0.15.1"
          imagePullPolicy: IfNotPresent
          args:
            - "--config.file=/config/blackbox.yaml"
          ports:
            - containerPort: 9115
          volumeMounts:
            - mountPath: /config
              name: prometheus-config
      volumes:
        - name: prometheus-config
          configMap:
            name: prometheus-blackbox-exporter

---
kind: Service
apiVersion: v1
metadata:
  name: prometheus-blackbox-exporter
  labels:
    app: prometheus-blackbox-exporter
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 9115
      protocol: TCP
  selector:
    app: prometheus-blackbox-exporter

---

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-blackbox-exporter
  labels:
    app: prometheus-blackbox-exporter
data:
  blackbox.yaml: |
    modules:
      http_2xx:
        http:
          no_follow_redirects: false
          preferred_ip_protocol: ip4
          valid_http_versions:
          - HTTP/1.1
          - HTTP/2
          valid_status_codes: []
        prober: http
        timeout: 5s

2. in prometheus update prometheus.yml file as below

3. Prometheus query

probe_http_status_code{job="web1"}

Rancher proxy rule in httpd with websocket secure (wss)

<VirtualHost *:80>
	ServerName rancher.initedit.com
	Redirect permanent / https://rancher.initedit.com/
	RewriteEngine on
	RewriteCond %{SERVER_NAME} =rancher.initedit.com [OR]
	RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,QSA,R=permanent]
</VirtualHost>

<VirtualHost *:443>
    ServerName rancher.initedit.com
    AllowEncodedSlashes on
    SSLEngine On
    SSLProxyEngine On
    RewriteEngine on
    SSLProxyVerify none
    SSLProxyCheckPeerCN off
    SSLProxyCheckPeerName off
    SSLProxyCheckPeerExpire off
    RequestHeader set X-Forwarded-Proto "https"
    RewriteCond %{HTTP:Upgrade} =websocket [NC]
    RewriteRule /(.*)   wss://192.168.0.183:8443/$1 [P,L]
    RewriteCond %{HTTP:Upgrade} !=websocket [NC]
    RewriteRule /(.*)   https://192.168.0.183:8443/$1 [P,L]
    ProxyPassReverse / https://192.168.0.183:8443/
    ProxyPreserveHost On
    SSLCertificateFile /etc/letsencrypt/live/rancher.initedit.com/cert.pem
    SSLCertificateKeyFile /etc/letsencrypt/live/rancher.initedit.com/privkey.pem
    SSLCertificateChainFile /etc/letsencrypt/live/rancher.initedit.com/fullchain.pem
    Include /etc/letsencrypt/options-ssl-apache.conf
</VirtualHost>

More info : https://stackoverflow.com/questions/27526281/websockets-and-apache-proxy-how-to-configure-mod-proxy-wstunnel

Install k8 with 2 nodes on centos7

Prerequisites:
– Disable swap
– Disable SElinux
– Disable Firewalld(optional if all k8 rule added)

Servers IP:
kmaster1 = 192.168.0.10
knode1 = 192.168.0.11
knode2 = 192.168.0.12

#Disable swap
swapoff -a

swapline=$(cat -n /etc/fstab | grep swap | awk '{print $1}')
if [ $(cat /etc/fstab | grep swap | awk '{print substr($0,0,1)}') != "#" ]
then
sed -i ""$swapline"s/.*/#&/" /etc/fstab
fi


#Disable SElinux
setenforce 0
sed -i  's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

#Disable firewalld
systemctl stop firewalld
systemctl disable firewalld

#iptables conf
echo 'net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1' > /etc/sysctl.d/k8.conf

sysctl --system

Add Kubernetes Repository

echo '[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
' >  /etc/yum.repos.d/kubernetes.repo

On kmaster01

yum install -y kubelet kubeadm kubectl docker
systemctl enable kubelet
systemctl start kubelet
systemctl enable docker
systemctl start docker

hostnamectl set-hostname kmaster01

kubeadm init

At end of above command run below command and save the join command.

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

AND note the join command

Apply weave network:

kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"

On knode1:

yum install -y kubelet kubeadm kubectl docker
systemctl enable kubelet
systemctl start kubelet
systemctl enable docker
systemctl start docker

hostnamectl set-hostname knode1

#Join command
kubeadm join 192.168.0.10:6443 --token lfh49o.f9na1435g8vs1fmt \
     --discovery-token-ca-cert-hash sha256:0064f08a4c0ef36e454b683f61a68e0bf78d9cdb45f7905128c68b28fc2a5b3e

On knode2:

yum install -y kubelet kubeadm kubectl docker
systemctl enable kubelet
systemctl start kubelet
systemctl enable docker
systemctl start docker

hostnamectl set-hostname knode2

#Join command
kubeadm join 192.168.0.10:6443 --token lfh49o.f9na1435g8vs1fmt \
     --discovery-token-ca-cert-hash sha256:0064f08a4c0ef36e454b683f61a68e0bf78d9cdb45f7905128c68b28fc2a5b3e

nginx-app.yml


apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-nginx
spec:
  selector:
    matchLabels:
      run: my-nginx
  replicas: 2
  template:
    metadata:
      labels:
        run: my-nginx
    spec:
      containers:
      - name: my-nginx
        image: nginx
        ports:
        - containerPort: 80

nginx-svc.yml

apiVersion: v1
kind: Service
metadata:
  name: nginx-svc
  labels:
    run: nginx-svc
spec:
  type: NodePort
  ports:
  - port: 80
    protocol: TCP
  selector:
    run: my-nginx

kubectl apply -f ngix-app.yml
kubectl apply -f ngix-svc.yml