https://blog.alexellis.io/test-drive-k3s-on-raspberry-pi/
# append to /boot/cmdline.txt
cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory
note this is not using docker, it uses containerd directly
curl -sfL https://get.k3s.io | sh -
# get install status with
sudo systemctl status k3s
# check node
sudo /usr/local/bin/kubectl get node
in case you’re a dummy like me and you spin up more than one master
/usr/local/bin/k3s-uninstall.sh
# on initial server
sudo cat /var/lib/rancher/k3s/server/node-token
# on each worker node
export K3S_URL="https://10.4.70.160:6443"
export K3S_TOKEN="K10c5f1e7496cd1fbfb797020ca59246a5875d8cf9364576322a8398e039bf7d249::server:f4eabc85d3249cca6a2620d971e6362d
"
curl -sfL https://get.k3s.io | sh -
# (repeat for all workers)
note this requires that you have an actual ssh key, not using gpg-agent and a yubikey as is my normal practice, so set one of those up as follows
ssh-keygen -o -t ed25519 -f k3s-vmware.key
ssh-copy-id -i k3s-vmware.key notch@k3s-server1
ssh-copy-id -i k3s-vmware.key notch@k3s-worker1
ssh-copy-id -i k3s-vmware.key notch@k3s-worker2
ssh-copy-id -i k3s-vmware.key notch@k3s-worker3
# from mac
brew install k3sup
export SERVER=10.4.70.160
export AGENT=10.4.70.161
k3sup install --ssh-key k3s-vmware.key --ip $SERVER --user notch
k3sup join --ssh-key k3s-vmware.key --ip $AGENT --server-ip $SERVER --user notch
# (repeat for $AGENT = 162,163)
note this may or may not work on centos 8
# k3sup install --ssh-key k3s-vmware.key --ip $SERVER --user notch
Running: k3sup install
Public IP: 10.4.70.160
[INFO] Finding release for channel v1.18
[INFO] Using v1.18.10+k3s2 as release
[INFO] Downloading hash https://github.com/rancher/k3s/releases/download/v1.18.10+k3s2/sha256sum-amd64.txt
[INFO] Skipping binary downloaded, installed k3s matches hash
Errors during downloading metadata for repository 'rancher-k3s-common-v1.18':
- Status code: 404 for https://rpm.rancher.io/k3s/v1.18/common/centos/8/noarch/repodata/repomd.xml (IP: 104.24.112.22)
https://blog.internetz.me/posts/kubernetes-k3s-cluster-using-k3sup-multi-master/
disable firewalls everywhere, centos support isnt quite right (cant reach cert service)
sudo systemctl stop firewalld
sudo systemctl disable firewalld
(you will get this error if you dont)
ERRO[2020-11-12T20:43:22.704801893-05:00] failed to get CA certs: Get "https://127.0.0.1:44141/cacerts": read tcp 127.0.0.1:34944->127.0.0.1:44141: read: connection reset by peer
make sure server node is healthy
sudo /usr/local/bin/kubectl get node -o wide
sudo /usr/local/bin/k3s kubectl get node
On a different node run the below. NODE_TOKEN comes from /var/lib/rancher/k3s/server/node-token
sudo k3s agent --server https://myserver:6443 --token ${NODE_TOKEN}
sudo /usr/local/bin/k3s agent --server https://10.4.70.160:6443 --token K10c5f1e7496cd1fbfb797020ca59246a5875d8cf9364576322a8398e039bf7d249::server:f4eabc85d3249cca6a2620d971e6362d
# test from mac
export KUBECONFIG=/Users/gnotch/src/k3s-vmware/k3s.yaml
kubectl get node -o wide
https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/
there is a newer version URL than listed on the official blog
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.4/aio/deploy/recommended.yaml
dashboard.admin-user.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kubernetes-dashboard
dashboard.admin-user-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kubernetes-dashboard
build 2 yaml configs and apply
kubectl apply -f dashboard.admin-user.yaml
kubectl apply -f dashboard.admin-user-role.yaml
get dashboard API token
kubectl -n kubernetes-dashboard describe secret admin-user-token
fire up proxy (port 8001 by default
kubectl proxy
dashboard will be available at this URL, and you will need the token above
kubectl get services
kubectl expose deployment/kubernetes-bootcamp –type=“NodePort” –port 8080
export NODE_PORT=$(kubectl get services/kubernetes-bootcamp -o go-template=‘{{(index .spec.ports 0).nodePort}}’) echo NODE_PORT=$NODE_PORT
label pods key=value
kubectl exec -it $POD curl localhost
LoadBalancer is only for cloud services?
kubectl delete service undos the expose part
look at the yaml for nginx
exposing traefik dashboard https://www.jurgenallewijn.nl/k3s-enable-traefik-dashboard/ ]
add to /var/lib/rancher/k3s/server/manifests/traefik.yaml on k3s-server (master)
dashboard:
enabled: true
(itll eventually start)
port forward dashboard to localhost
kubectl -n kube-system port-forward deployment/traefik 8080
kubectl get endpoints -n kube-system
https://github.com/rancher/k3d/issues/103
kubectl -n kube-system edit service traefik
https://rancher.com/docs/rancher/v2.x/en/installation/install-rancher-on-k8s/
helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
kubectl create namespace cattle-system
(do cert manager install below here)
helm install rancher rancher-latest/rancher
–namespace cattle-system
–set hostname=k3s-rancher.winternotch.com
(watch status)
kubectl -n cattle-system rollout status deploy/rancher
(something broken here, only mapped to one of the k3s nodes… going for full rancher install)
kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.crds.yaml
kubectl create namespace cert-manager
helm repo add jetstack https://charts.jetstack.io
helm repo update
helm install \
cert-manager jetstack/cert-manager \
--namespace cert-manager \
--version v1.0.4
list all pods
kubectl get pods --all-namespaces
get cluster API endpoint (if not using proxy)
kubectl config view -o jsonpath='{"Cluster name\tServer\n"}{range .clusters[*]}{.name}{"\t"}{.cluster.server}{"\n"}{end}'
set up kubectl localhost proxy (port 8001 by default)
kubectl proxy
run command on pod > kubectl exec -it $POD curl localhost
get endpoints > kubectl get endpoints -n kube-system
get services > kubectl get services
describe services > kubectl describe svc nginx-service
https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/
api is available at /openapi/v2 for Postman
https://kubernetes.io/docs/reference/using-api/
https://kubernetes.io/docs/concepts/overview/kubernetes-api/
.sops.yaml
creation_rules:
- pgp: >-
83924F50D1E25F77A4B6C9CDAD8030EBA885F318
OPTIONAL: create a helper for sublime to call subl in blocking mode
#!/bin/bash
/Users/gnotch/bin/subl -w $1
OPTIONAL: set up editor env var (if you dont want to use VI)
export EDITOR=/Users/gnotch/bin/subl_blocking
which leverages https://github.com/tiangolo/uwsgi-nginx-flask-docker
jinja templates https://jinja.palletsprojects.com/en/2.11.x/templates/
cheatsheet
https://kubernetes.io/docs/reference/kubectl/cheatsheet/
rasperry pi links https://alexellisuk.medium.com/walk-through-install-kubernetes-to-your-raspberry-pi-in-15-minutes-84a8492dc95a
https://amithm.ca/2020/10/kubernetes-raspberrypi-homelab/
k3sup HA https://ma.ttias.be/deploying-highly-available-k3s-k3sup/
domesticating kubernetes https://blog.quickbird.uk/domesticating-kubernetes-d49c178ebc41