Greylog setup with filebeat and Raw/Plaintext TCP

More : https://docs.graylog.org/en/4.0/pages/installation/docker.html

version: '2'
services:
  mongodb:
    image: mongo:4.2
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
    environment:
      - http.host=0.0.0.0
      - transport.host=localhost
      - network.host=0.0.0.0
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    mem_limit: 1g
  graylog:
    image: graylog/graylog:4.0
    environment:
      - GRAYLOG_PASSWORD_SECRET=somepasswordpepper
      - GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
      - GRAYLOG_HTTP_EXTERNAL_URI=http://192.168.0.228:9000/
    entrypoint: /usr/bin/tini -- wait-for-it elasticsearch:9200 --  /docker-entrypoint.sh
    links:
      - mongodb:mongo
      - elasticsearch
    restart: always
    depends_on:
      - mongodb
      - elasticsearch
    ports:
      # Graylog web interface and REST API
      - 9000:9000
      # Syslog TCP
      - 1514:1514
      # Syslog UDP
      - 1514:1514/udp
      # GELF TCP
      - 12201:12201
      # GELF UDP
      - 12201:12201/udp
      - 5555:5555
      - 5551:5551

Note : 5555 port will be used for filebeat as logstash output

System / inputs >> select input > Beats > launch input > global tick > titile > port = 5555 > threads =2 > Save

  • Edit /etc/filebeat/filebeat.yml

Output:

Raw/Plaintext TCP

System / inputs >> select input > Raw/Plaintext TCP > launch input > global tick > titile > port = 5551 > threads =2 > Save

for sending Raw/Plaintext TCP

echo "this is log" > /dev/tcp/grelog_server/5551

haproxy context based routing

http://192.168.0.228:8080/app1 => http://192.168.0.228:8081
http://192.168.0.228:8080/app2 => http://192.168.0.228:8082

haproxy.cfg

global
    daemon
    maxconn 256

defaults
    timeout connect 10s
    timeout client 30s
    timeout server 30s
    mode http
    maxconn 3000

frontend http_in
    bind *:8080
    use_backend app1_backend if { path /app1 }
    use_backend app2_backend if { path /app2 }

backend app1_backend
    http-request set-path %[path,regsub(^/app1/?,/)]
    server server1 192.168.0.228:8081

backend app2_backend
    http-request set-path %[path,regsub(^/app2/?,/)]
    server server1 192.168.0.228:8082

more : https://grafana.com/tutorials/run-grafana-behind-a-proxy/#configure-haproxy

raspberry pi as kubernetes worker node

  • Install docker
apt-get install \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg \
    lsb-release

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

echo \
  "deb [arch=arm64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

apt-get update

apt-get install docker-ce docker-ce-cli containerd.io

https://docs.docker.com/engine/install/ubuntu/

echo 'cgroup_memory=1' > /boot/cmdline.txt


echo '{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
' > /etc/docker/daemon.json



sudo sed -i '$ s/$/ cgroup_enable=cpuset cgroup_enable=memory cgroup_memory=1 swapaccount=1/' /boot/firmware/cmdline.txt


cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sudo sysctl --system
  • Install kubernetes component
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -

cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main
EOF

apt update && sudo apt install -y kubelet kubeadm kubectl


echo '[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cgroup-driver=systemd"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS' > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf


systemctl daemon-reload

systemctl restart kubelet

https://opensource.com/article/20/6/kubernetes-raspberry-pi

https://stackoverflow.com/questions/45708175/kubelet-failed-with-kubelet-cgroup-driver-cgroupfs-is-different-from-docker-c