How to set up a cheap Kubernetes cluster on Google Cloud Platform

How to set up a cheap Kubernetes cluster on Google Cloud Platform

Check account

1
gcloud auth list
1
ACCOUNT=___
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
PROJECT_NAME=cocotola-001
PROJECT_ID=cocotola-001
ZONE=asia-northeast1-c
REGION=asia-northeast1
CONTEXT_NAME=cocotola-001
CLUSTER_NAME=cocotola-001
TRAEFIK_IP_NAME=traefik
CLOUD_NAT_IP_NAME=cloud-nat
ZONE_NAME=kujilabo
HOST_NAME=foo.kujilabo.com
DNS_NAME=kujilabo.com
ROUTER_NAME=my-nat-router
NAT_NAME=my-nat-config
NETWORK_NAME=my-net
SUBNET_NAME=my-subnet
TRAEFIK_VM_NAME=traefik
TRAEFIK_DNS_SA_NAME=traefik-dns

1
2
3
4
gcloud config configurations create $PROJECT_NAME
gcloud config set project $PROJECT_ID
gcloud config set account $ACCOUNT
gcloud config set compute/zone $ZONE
1
2
3
PROJECT_ID=$(gcloud config get-value project)
PROJECT_NUMBER="$(gcloud projects describe ${PROJECT_ID} \
  --format='get(projectNumber)')"

Enable APIs

1
2
3
gcloud services enable compute.googleapis.com
gcloud services enable dns.googleapis.com
gcloud services enable container.googleapis.com

Reserve IP Addresses

1
2
gcloud compute addresses create $TRAEFIK_IP_NAME   --region $REGION
gcloud compute addresses create $CLOUD_NAT_IP_NAME --region $REGION

Check IP Addresses and set env

1
gcloud compute addresses list
1
2
TRAEFIK_IP_VALUE=
CLOUD_NAT_IP_VALUE=

Create network

1
gcloud compute networks create $NETWORK_NAME --subnet-mode custom
1
2
3
4
5
6
gcloud compute networks subnets create $SUBNET_NAME \
  --network $NETWORK_NAME \
  --region $REGION \
  --range 10.146.0.0/29 \
  --secondary-range my-pods=10.4.0.0/14,my-services=10.0.32.0/20 \
  --enable-private-ip-google-access

Create firewall rules

1
2
3
4
gcloud compute firewall-rules create allow-ssh    --network $NETWORK_NAME --allow tcp:22
gcloud compute firewall-rules create http-server  --network $NETWORK_NAME --allow tcp:80
gcloud compute firewall-rules create https-server --network $NETWORK_NAME --allow tcp:443
gcloud compute firewall-rules create ingress-8443 --network $NETWORK_NAME --allow tcp:8443 --source-ranges 172.16.0.32/28

Set up Cloud NAT

1
gcloud compute routers create $ROUTER_NAME --network=$NETWORK_NAME --region=$REGION
1
2
3
4
5
gcloud compute routers nats create $NAT_NAME \
  --router=$ROUTER_NAME \
  --region=$REGION \
  --nat-all-subnet-ip-ranges \
  --nat-external-ip-pool=$CLOUD_NAT_IP_NAME

Create DNS records

1
2
3
4
gcloud dns managed-zones create $ZONE_NAME \
  --description=$DNS_NAME \
  --dns-name=$DNS_NAME \
  --visibility=public
1
gcloud dns record-sets transaction start --zone="$ZONE_NAME"
1
2
3
4
5
gcloud dns record-sets transaction add $TRAEFIK_IP_VALUE \
  --name="$HOST_NAME" \
  --ttl="300" \
  --type="A" \
  --zone="$ZONE_NAME"
1
gcloud dns record-sets transaction execute --zone="$ZONE_NAME"
1
gcloud dns record-sets list --zone="$ZONE_NAME"

Put nameservers into GoogleDomains

Create service account for DNS challenge

1
2
3
4
gcloud iam service-accounts create $TRAEFIK_DNS_SA_NAME
gcloud projects add-iam-policy-binding $PROJECT_ID \
   --member serviceAccount:$TRAEFIK_DNS_SA_NAME@$PROJECT_ID.iam.gserviceaccount.com \
   --role roles/dns.admin
1
2
gcloud iam service-accounts keys create $TRAEFIK_DNS_SA_NAME.json \
  --iam-account $TRAEFIK_DNS_SA_NAME@$PROJECT_ID.iam.gserviceaccount.com

Create VM instance for traefik

1
2
3
4
gcloud compute instances create $TRAEFIK_VM_NAME \
  --machine-type=e2-micro \
  --network-interface=network-tier=PREMIUM,subnet=$SUBNET_NAME,private-network-ip=10.146.0.2,address=$TRAEFIK_IP_VALUE \
  --tags=http-server,https-server

Install traefik on VM

https://gist.github.com/hirohara4913/3c188b1e5fac050f810216d20bbb5b28

1
2
3
4
5
6
7
sudo apt install -y wget
wget https://github.com/traefik/traefik/releases/download/v2.5.4/traefik_v2.5.4_linux_amd64.tar.gz
tar xvzf traefik_v2.5.4_linux_amd64.tar.gz
sudo mv ./traefik /usr/local/bin
sudo chown root:root /usr/local/bin/traefik
sudo chmod 755 /usr/local/bin/traefik
sudo setcap 'cap_net_bind_service=+ep' /usr/local/bin/traefik

Create OS user

1
2
3
4
5
6
sudo groupadd -g 500 traefik
sudo useradd \
  -g traefik --no-user-group \
  --home-dir /var/www --no-create-home \
  --shell /usr/sbin/nologin \
  --system --uid 500 traefik

Make directories

1
2
3
4
5
6
7
8
sudo mkdir /etc/traefik
sudo mkdir /etc/traefik/acme
sudo chown -R root:root /etc/traefik
sudo chown -R traefik:traefik /etc/traefik/acme
sudo mkdir /var/log/traefik
sudo chown -R traefik:traefik /var/log/traefik
sudo mkdir -p /usr/local/var/traefik
sudo chown -R traefik:traefik /usr/local/var/traefik

Make configuration files

1
2
3
4
vi traefik.toml
vi traefik.route.toml
vi traefik.service
vi traefik-dns.json
1
2
3
sudo mv ./traefik.toml /etc/traefik/
sudo mv ./traefik.route.toml /etc/traefik/
sudo mv ./traefik-dns.json /etc/traefik/

Register service for traefik.

1
2
3
4
sudo mv ./traefik.service /etc/systemd/system/
sudo chown root:root /etc/systemd/system/traefik.service
sudo chmod 644 /etc/systemd/system/traefik.service
sudo systemctl daemon-reload
1
sudo systemctl start traefik.service

Set up log rotation.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
sudo tee /etc/logrotate.d/traefik <<EOF >/dev/null
/var/log/traefik/traefik.log {
  daily
  rotate 30
  missingok
  notifempty
  copytruncate
  dateext
  dateformat %Y-%m-%d
}

/var/log/traefik/access.log {
  daily
  rotate 30
  missingok
  notifempty
  copytruncate
  dateext
  dateformat %Y-%m-%d
}
EOF
1
sudo /usr/sbin/logrotate /etc/logrotate.conf

Create kubernetes cluster

1
2
3
4
5
6
7
8
9
10
11
gcloud container clusters create $CLUSTER_NAME \
  --preemptible \
  --machine-type=e2-small \
  --disk-size=10 \
  --enable-autoscaling --num-nodes=2 --max-nodes=3 \
  --no-enable-master-authorized-networks \
  --enable-ip-alias \
  --enable-private-nodes \
  --network $NETWORK_NAME \
  --subnetwork $SUBNET_NAME \
  --master-ipv4-cidr 172.16.0.32/28
1
2
CLUSTER_NAME=`kubectl config current-context`
kubectl config rename-context $CLUSTER_NAME $CONTEXT_NAME

Create ingress controller

https://gist.github.com/hirohara4913/65246a22d2520b17b3686b12455426c1

1
kubectl apply -f deploy.yaml

Deploy web application

1
2
kubectl create deployment web --image=gcr.io/google-samples/hello-app:1.0
kubectl expose deployment web --type=NodePort --port=8080

Create Ingress resource

https://gist.github.com/hirohara4913/d3d2b0e381fd369fd5c00f983ef919ed

1
kubectl apply -f web.yaml