Setting up multinode K8s Cluster

1. Changing kernal parameters for packet routing

vi /etc/sysctl.conf

Add this lines

net.ipv4.ip_forward=1
net.ipv6.conf.all.forwarding=1

View using

sysctl -p

2. Disable swap

vi /etc/fstab

and comment swap

Directly disabling from cmd and view

swapoff -a
free -m

3. Setting up containerd

apt update
apt upgrade -y

Cleaning if previously installed

sudo apt purge -y containerd
sudo rm -rf /etc/containerd /var/lib/containerd

Install correct containerd (official Docker build)

sudo apt update
sudo apt install -y containerd.io

If not found, add Docker repo:

sudo apt install -y ca-certificates curl gnupg
sudo install -m 0755 -d /etc/apt/keyrings

curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg

echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
  $(. /etc/os-release && echo $VERSION_CODENAME) stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

sudo apt update
sudo apt install -y containerd.io

Generate config-file

sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

Fix cgroup driver (MOST IMPORTANT)

sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml

or find "SystemdCgroup = false" section and make it "SystemdCgroup = true"

Restarting containerd

sudo systemctl daemon-reexec
sudo systemctl enable --now containerd
sudo systemctl restart containerd

Verify using

crictl info

4. Adding kubeadm, kubeelet and kubectl repos and installing in all nodes

sudo apt-get update
# apt-transport-https may be a dummy package; if so, you can skip that package
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
# If the directory `/etc/apt/keyrings` does not exist, it should be created before the curl command, read the note below.
# sudo mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
# This overwrites any existing configuration in /etc/apt/sources.list.d/kubernetes.list
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo systemctl enable --now kubelet
kubeadm config images pull

5. Kubeadm init and Joining the nodes

vi kubeadm-config.yaml

add this

---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
networking:
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/16
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: <YOUR_NODE'S/MACHINES_IP>
  bindPort: 6443
nodeRegistration:
  kubeletExtraArgs:
    node-ip: <YOUR_NODE'S/MACHINES_IP>
kubeadm init --config=kubeadm-config.yaml

To make kubectl command available

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

Installing calico as CNI

kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml

Kubeadm join token create

kubeadm token create --print-join-command

To view and disable the taints on a node

kubectl describe node NODE_NAME | grep Taints
kubectl taint node CONTROL_PLANE_NODE_NAME node-role.kubernetes.io/control-plane:NoSchedule-

Joining using config-file

vi kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration
discovery:
  bootstrapToken:
    apiServerEndpoint: <YOUR_NODE'S/MACHINES_IP>:6443
    token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
    caCertHashes:
      - "sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
nodeRegistration:
  kubeletExtraArgs:
   node-ip: <YOUR_NODE'S/MACHINES_IP>

Testing

kubectl get nodes
kubectl get all -A
kubectl create deployment nginx --image=nginx
kubectl expose deployment nginx --port=80 --type=NodePort
kubectl get svc nginx
kubectl get pods -o wide

Debugging

kubectl describe pod nginx
kubectl logs nginx

6. Cleaning up after kubeadm reset

Resting kubeadm

kubeadm reset
rm -rf /etc/cni/net.d
sudo rm -rf $HOME/.kube

Remove Kubernetes packages

apt-get purge -y kubeadm kubectl kubelet kubernetes-cni 
apt-get autoremove -y

Remove containerd

apt-get purge -y containerd

Remove Kubernetes and CNI directories

rm -rf /etc/kubernetes/
rm -rf /var/lib/etcd
rm -rf /var/lib/kubelet
rm -rf /etc/cni/
rm -rf /var/lib/cni/
rm -rf /opt/cni/

Clean iptables

iptables -F || true
iptables -t nat -F || true
iptables -t mangle -F || true
iptables -X || true

Clean nftables (if present)

nft flush ruleset || true

Delete CNI interfaces

ip link delete cni0 || true
ip link delete flannel.1 || true

Remove CRI config

rm -f /etc/crictl.yaml