# Get the NGINX service external LB IP
kubcetl get service
# Curl to test the Ingress
curl -v -k --resolve zaid.cloud.com:443:<LB-External-IP>https://zaid.cloud.com
Recently encountered a scenario post cutover where clients were getting 403 forbidden errors when trying to reach the Application hosted behind Application Gateway. We already had the client IP whitelisted on Application Gateway WAF. Log Analytics really came to our rescue.
We had the Application Gateway already integrated with Log Analytics.
AGW – Diagnostics Log Analytics
We started by looking at the Application Gateway Access logs to check which API was returning 403 error
AGW Access Log
// Errors by URI
// Number of errors by URI.
// To create an alert for this query, click '+ New alert rule'
AzureDiagnostics
| where ResourceType == "APPLICATIONGATEWAYS" and OperationName == "ApplicationGatewayAccess" and httpStatus_d == 403
| summarize AggregatedValue = count() by requestUri_s, _ResourceId
| sort by AggregatedValue desc
Next, we pulled all the Blocked records for the request URI from Application Gateway Firewall Log
AGW Firewall Log
AzureDiagnostics
| where ResourceProvider == "MICROSOFT.NETWORK" and Category == "ApplicationGatewayFirewallLog"
| where action_s contains "Blocked"
| where requestUri_s contains "/manager/html"
Add transactionId_g column in the result
Add transactionId_g column
Scroll to the right of results view to get the transactionId_g for the forbidden error
View transactionId_g
Run the below query to get the detailed message for the Transaction id
Detailed Firewall Log
AzureDiagnostics
| where ResourceProvider == "MICROSOFT.NETWORK" and Category == "ApplicationGatewayFirewallLog"
| where action_s contains "Blocked"
| where requestUri_s contains "/manager/html"
| where transactionId_g contains "e7c58a61-e42f-3063-0a24-90f3a2d01044"
Post this we were able to go ahead and provide feedback to the client on OWASP issues.
Other Queries:
AzureDiagnostics
| where ResourceProvider == "MICROSOFT.NETWORK" and Category == "ApplicationGatewayFirewallLog"
| where action_s contains "Blocked"
| where TimeGenerated > ago(48h)
| sort by TimeGenerated desc
| where policyId_s == "<>"
| summarize count() by clientIp_s
AzureDiagnostics
| where ResourceProvider == "MICROSOFT.NETWORK" and Category == "ApplicationGatewayFirewallLog"
| where action_s contains "Blocked"
| where TimeGenerated > ago(0.15h)
| sort by TimeGenerated desc
| where policyId_s == "<>"
Save cost on a development cluster. Stop your control plane and agent nodes altogether, allowing you to save on all the compute costs, while maintaining all your objects and cluster state stored for when you start it again.
Stop the Cluster
az aks stop --name zcAKSCluster --resource-group zcResourceGroup
Start the Cluster
az aks start --name zcAKSCluster --resource-group zcResourceGroup
Validate the state
az aks show --name zcAKSCluster --resource-group zcResourceGroup
Azure Arc-enabled Kubernetes lets you make your on-premises or cloud Kubernetes cluster visible to App Service, Functions, and Logic Apps in Azure. You can create an app and deploy it just like another Azure region.
Note: In this demo, I have used AKS for demonstrating the feature. In the production scenario, you would implement this for other K8s clusters.
Onboard the K8s cluster to Azure Arc
#======================================
######Create a connected cluster
#======================================
#Create a cluster
aksClusterGroupName="rg-k8s-meetup-01" # Name of resource group for the AKS cluster
aksName="arc-k8s-meetup" # Name of the AKS cluster
resourceLocation="eastus" # "eastus" or "westeurope"
az group create -g $aksClusterGroupName -l $resourceLocation
az aks create --resource-group $aksClusterGroupName --name $aksName --enable-aad --generate-ssh-keys
infra_rg=$(az aks show --resource-group $aksClusterGroupName --name $aksName --output tsv --query nodeResourceGroup)
az network public-ip create --resource-group $infra_rg --name MyPublicIP --sku STANDARD
staticIp=$(az network public-ip show --resource-group $infra_rg --name MyPublicIP --output tsv --query ipAddress)
#Get the Kubeconfig file
az aks get-credentials --resource-group $aksClusterGroupName --name $aksName --admin
kubectl get ns
#Create RG for Azure Arc resources
groupName="rg-arc-meetup-01" # Name of resource group for the connected cluster
az group create -g $groupName -l $resourceLocation
#Connect the CLuster
clusterName="arc-k8s-cluster-meetup" # Name of the connected cluster resource
az connectedk8s connect --resource-group $groupName --name $clusterName
az connectedk8s show --resource-group $groupName --name $clusterName
Connected Cluster
App Service Extension
#======================================
######Create a Log Analytics workspace
#======================================
workspaceName="$groupName-workspace" # Name of the Log Analytics workspace
az monitor log-analytics workspace create \
--resource-group $groupName \
--workspace-name $workspaceName
logAnalyticsWorkspaceId=$(az monitor log-analytics workspace show \
--resource-group $groupName \
--workspace-name $workspaceName \
--query customerId \
--output tsv)
logAnalyticsWorkspaceIdEnc=$(printf %s $logAnalyticsWorkspaceId | base64 -w0)
logAnalyticsKey=$(az monitor log-analytics workspace get-shared-keys \
--resource-group $groupName \
--workspace-name $workspaceName \
--query primarySharedKey \
--output tsv)
logAnalyticsKeyEnc=$(printf %s $logAnalyticsKey | base64 -w0)
#======================================
######Install the App Service extension
#======================================
extensionName="appservice-ext" # Name of the App Service extension
namespace="appservice-ns" # Namespace in your cluster to install the extension and provision resources
kubeEnvironmentName="appservice-kube" # Name of the App Service Kubernetes environment resource
az k8s-extension create \
--resource-group $groupName \
--name $extensionName \
--cluster-type connectedClusters \
--cluster-name $clusterName \
--extension-type 'Microsoft.Web.Appservice' \
--release-train stable \
--auto-upgrade-minor-version true \
--scope cluster \
--release-namespace $namespace \
--configuration-settings "Microsoft.CustomLocation.ServiceAccount=default" \
--configuration-settings "appsNamespace=${namespace}" \
--configuration-settings "clusterName=${kubeEnvironmentName}" \
--configuration-settings "loadBalancerIp=${staticIp}" \
--configuration-settings "keda.enabled=true" \
--configuration-settings "buildService.storageClassName=default" \
--configuration-settings "buildService.storageAccessMode=ReadWriteOnce" \
--configuration-settings "customConfigMap=${namespace}/kube-environment-config" \
--configuration-settings "envoy.annotations.service.beta.kubernetes.io/azure-load-balancer-resource-group=${aksClusterGroupName}" \
--configuration-settings "logProcessor.appLogs.destination=log-analytics" \
--configuration-protected-settings "logProcessor.appLogs.logAnalyticsConfig.customerId=${logAnalyticsWorkspaceIdEnc}" \
--configuration-protected-settings "logProcessor.appLogs.logAnalyticsConfig.sharedKey=${logAnalyticsKeyEnc}"
extensionId=$(az k8s-extension show \
--cluster-type connectedClusters \
--cluster-name $clusterName \
--resource-group $groupName \
--name $extensionName \
--query id \
--output tsv)
az resource wait --ids $extensionId --custom "properties.installState!='Pending'" --api-version "2020-07-01-preview"
kubectl get pods -n $namespace
App Service Extension
Custom Location
#======================================
######Create a custom location
#======================================
customLocationName="reactor-meetup-location" # Name of the custom location
connectedClusterId=$(az connectedk8s show --resource-group $groupName --name $clusterName --query id --output tsv)
#Create the custom location
az customlocation create \
--resource-group $groupName \
--name $customLocationName \
--host-resource-id $connectedClusterId \
--namespace $namespace \
--cluster-extension-ids $extensionId
#Validate
az customlocation show --resource-group $groupName --name $customLocationName
customLocationId=$(az customlocation show \
--resource-group $groupName \
--name $customLocationName \
--query id \
--output tsv)
Custom Location
App Service Kubernetes Environment
#======================================
######Create the App Service Kubernetes environment
#======================================
az appservice kube create \
--resource-group $groupName \
--name $kubeEnvironmentName \
--custom-location $customLocationId \
--static-ip $staticIp
#Validate
az appservice kube show --resource-group $groupName --name $kubeEnvironmentName
App Service Kubernetes Environment
Web App on Azure Arc
#======================================
######Create a web app on Azure Arc
#======================================
groupName="rg-arc-meetup-01"
appServicePlan="arc-appservice-plan"
appService="webapponarc"
#Create an App Service plan
az appservice plan create -g $groupName -n $appServicePlan --custom-location $customLocationName --per-site-scaling --is-linux --sku K1
#Create an app
az webapp create --plan $appServicePlan --resource-group $groupName --name $appService --custom-location $customLocationName --runtime 'NODE|12-lts'
#Deploy Sample Code
az webapp deployment source config-zip --resource-group $groupName --name $appService --src package.zip