karpenter-lab¶

compatibility¶

install¶

https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/

using eksdemo¶

https://github.com/awslabs/eksdemo/blob/main/docs/install-karpenter.md
```
eksdemo install karpenter -c ekscluster --dry-run
```

using helm¶

create sqs for interrupt event

aws sqs  create-queue --queue-name sqs-${CLUSTER_NAME}

create service account (refer: func-create-iamserviceaccount)

install

echo ${KARPENTER_VERSION:=1.0.0}
echo ${KARPENTER_NAMESPACE:=karpenter}
echo ${CLUSTER_NAME:=ekscluster1}

helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter \
    --version "${KARPENTER_VERSION}" \
    --namespace "${KARPENTER_NAMESPACE}" --create-namespace  \
    --set "settings.clusterName=${CLUSTER_NAME}"   \
    --set "settings.interruptionQueue=sqs-${CLUSTER_NAME}"   \
    --set controller.resources.requests.cpu=1   \
    --set controller.resources.requests.memory=1Gi   \
    --set controller.resources.limits.cpu=1   \
    --set controller.resources.limits.memory=1Gi   \
    --set serviceAccount.create=false \
    --set serviceAccount.name=karpenter

helm upgrade -i -f a.value.yaml karpenter oci://public.ecr.aws/karpenter/karpenter \
    --version "${KARPENTER_VERSION}" \
    --namespace "${KARPENTER_NAMESPACE}" --create-namespace

previous version¶

echo ${CLUSTER_NAME:=eks-upgrade-demo}
echo ${AWS_DEFAULT_REGION:=us-east-2}

export KARPENTER_VERSION=v0.27.5
export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)"
export TEMPOUT=$(mktemp)

echo $KARPENTER_VERSION $CLUSTER_NAME $AWS_DEFAULT_REGION $AWS_ACCOUNT_ID $TEMPOUT

curl -fsSL https://karpenter.sh/"${KARPENTER_VERSION}"/getting-started/getting-started-with-karpenter/cloudformation.yaml  > $TEMPOUT \
&& aws cloudformation deploy \
  --stack-name "Karpenter-${CLUSTER_NAME}" \
  --template-file "${TEMPOUT}" \
  --capabilities CAPABILITY_NAMED_IAM \
  --parameter-overrides "ClusterName=${CLUSTER_NAME}"

instance family¶

spec:
  requirements:

  - key: karpenter.k8s.aws/instance-family
    operator: In
    values: [c5, m5, r5]

install eks-node-viewer¶

go install github.com/awslabs/eks-node-viewer/cmd/eks-node-viewer@latest
sudo mv -v ~/go/bin/eks-node-viewer /usr/local/bin

scenario¶

coredns¶

kubectl get deploy -n kube-system coredns -o yaml

apiVersion: apps/v1
kind: Deployment
...
  template:
    metadata:
      annotations:
        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
        karpenter.sh/do-not-evict: "true"
      creationTimestamp: null
      labels:
        eks.amazonaws.com/component: coredns
        foo: bar
        k8s-app: kube-dns
        ...

workshop¶

run gpu pod¶

https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/

refer¶

ec2-spot-instance

sample¶

---
apiVersion: karpenter.k8s.aws/v1
kind: EC2NodeClass
metadata:
  name: default
spec:
  amiSelectorTerms:

    - alias: al2023@latest
  role: KarpenterNodeRole-ekscluster1
  securityGroupSelectorTerms:
  - tags:
      aws:eks:cluster-name: ekscluster1
  subnetSelectorTerms:
  - tags:
      Name: eksctl-ekscluster1-cluster/SubnetPrivate*
  tags:
    intent: apps

sample - no amd¶

https://docs.aws.amazon.com/eks/latest/userguide/create-node-pool.html

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      nodeSelector:
        kubernetes.io/arch: amd64
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:

            - matchExpressions:
              - key: eks.amazonaws.com/instance-cpu-manufacturer
                operator: In
                values:
                - aws
                - intel
      containers:
      - name: my-app
        image: nginx:latest