Kizz services have a requirement to manage certificates for the on-prem Kubernetes pods. Currently they use Kubernetes secrets to store the certificate. In the below setup we will evaluate AKV secret provider extension for Azure Arc enabled Kubernetes.
Pre-requisites:
– Azure Subscription
– Azure Arc connected Kubernetes cluster
– az cli version 2.25 or above
– SPN with contributor rights
- Open shell and connect to your Kubernetes cluster. I am connected to CAPI Kubernetes cluster
- Update below shell script parameters and run . ./keyvault_k8s_arc.sh
- The script will create the Key Vault and deploy a self signed certificate as per the parameters
- The script also deploys sample nginx app and ingress resources.
- Get the deployed pods and secrets
- Get the External IP of the Ingress Controller
- Test Ingress with TLS
Script
#!/bin/bash
# <--- Change the following environment variables according to your Azure service principal name --->
export appId='<Your Azure service principal name>'
export password='<Your Azure service principal password>'
export tenantId='<Your Azure tenant ID>'
export keyVaultResourceGroup='<KeyVault Resource Group name>'
export keyVaultLocation='<Key Vault Location>'
export host='hello.arc.com'
export certname='ingress-cert'
export namespace='hello-arc'
export resourceGroup='arc-capi-azure'
export arcClusterName='arc-capi-azure'
export k8sExtensionName='akvsecretsprovider'
export keyVaultName=secret-store-$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 10 | head -n 1)
# Installing Helm 3
echo "Installing Helm 3"
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh
echo "Login to Az CLI using the service principal"
az login --service-principal --username $appId --password $password --tenant $tenantId
echo "Checking if you have up-to-date Azure Arc Az CLI 'connectedk8s' extension..."
az extension show --name "connectedk8s" &> extension_output
if cat extension_output | grep -q "not installed"; then
az extension add --name "connectedk8s"
rm extension_output
else
az extension update --name "connectedk8s"
rm extension_output
fi
echo ""
echo "Checking if you have up-to-date Azure Arc Az CLI 'k8s-extension' extension..."
az extension show --name "k8s-extension" &> extension_output
if cat extension_output | grep -q "not installed"; then
az extension add --name "k8s-extension"
rm extension_output
else
az extension update --name "k8s-extension"
rm extension_output
fi
echo ""
# Create Key Vault and Import Certificate
echo "Creating Key Vault"
az group create --name $keyVaultResourceGroup --location $keyVaultLocation
az keyvault create --name $keyVaultName --resource-group $keyVaultResourceGroup --location $keyVaultLocation
echo "Generating a TLS Certificate"
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ingress-tls.key -out ingress-tls.crt -subj "/CN=${host}/O=${host}"
openssl pkcs12 -export -in ingress-tls.crt -inkey ingress-tls.key -out $certname.pfx -passout pass:
echo "Importing the TLS certificate to Key Vault"
az keyvault certificate import --vault-name $keyVaultName -n $certname -f $certname.pfx
echo "Create Azure Key Vault Kubernetes extension instance"
az k8s-extension create --name $k8sExtensionName --extension-type Microsoft.AzureKeyVaultSecretsProvider --scope cluster --cluster-name $arcClusterName --resource-group $resourceGroup --cluster-type connectedClusters --release-train preview --release-namespace kube-system --configuration-settings 'secrets-store-csi-driver.enableSecretRotation=true' 'secrets-store-csi-driver.syncSecret.enabled=true'
# Create a namespace for app and ingress resources
kubectl create ns $namespace
# Create the Kubernetes secret with the service principal credentials
kubectl create secret generic secrets-store-creds --namespace $namespace --from-literal clientid=${appId} --from-literal clientsecret=${password}
kubectl --namespace $namespace label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
# Deploy SecretProviderClass
echo "Creating Secret Provider Class"
cat <<EOF | kubectl apply -n $namespace -f -
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: azure-kv-sync-tls
spec:
provider: azure
secretObjects: # secretObjects defines the desired state of synced K8s secret objects
- secretName: ingress-tls-csi
type: kubernetes.io/tls
data:
- objectName: $certname
key: tls.key
- objectName: $certname
key: tls.crt
parameters:
usePodIdentity: "false"
keyvaultName: $keyVaultName
objects: |
array:
- |
objectName: $certname
objectType: secret
tenantId: $tenantId
EOF
# Deploy Ingress Controller
echo "Deploy ingress controller"
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update
helm install ingress-nginx/ingress-nginx --generate-name \
--namespace $namespace \
--set controller.replicaCount=2 \
--set controller.nodeSelector."beta\.kubernetes\.io/os"=linux \
--set defaultBackend.nodeSelector."beta\.kubernetes\.io/os"=linux \
-f - <<EOF
controller:
extraVolumes:
- name: secrets-store-inline
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: "azure-kv-sync-tls"
nodePublishSecretRef:
name: secrets-store-creds
extraVolumeMounts:
- name: secrets-store-inline
mountPath: "/mnt/secrets-store"
readOnly: true
EOF
# Checking if Ingress Controller is ready
echo "Waiting for Ingress Controller to be ready"
kubectl wait --namespace $namespace --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=120s
# Deploy Application
echo "Deploying the Application"
cat <<EOF | kubectl apply -n $namespace -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-app
labels:
app: nginx-app
spec:
replicas: 1
selector:
matchLabels:
app: nginx-app
template:
metadata:
labels:
app: nginx-app
spec:
containers:
- name: demo-app
image: nginx
---
apiVersion: v1
kind: Service
metadata:
name: nginx-app
spec:
type: ClusterIP
ports:
- port: 80
selector:
app: nginx-app
EOF
# Deploy an Ingress Resource referencing the Secret created by the CSI driver
echo "Deploying Ingress Resource"
cat <<EOF | kubectl apply -n $namespace -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-tls
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/rewrite-target: /$1
spec:
tls:
- hosts:
- $host
secretName: ingress-tls-csi
rules:
- host: $host
http:
paths:
- pathType: Prefix
backend:
service:
name: nginx-app
port:
number: 80
path: /(.*)
EOF
Zaid's Cloud 