💰FinOps & Cloud Cost Engineering

# Query via kubectl-cost (CLI) — daily cost per namespace
kubectl cost namespace \
  --window 7d \
  --show-cpu --show-memory --show-network --show-efficiency \
  --show-pv

# Query via Kubecost HTTP API — last 30 days, by deployment
curl "http://localhost:9090/model/allocation?window=30d&aggregate=deployment&accumulate=false" \
  | jq '.data[0] | to_entries | sort_by(-.value.totalCost) | .[:10]'

# Get cost per label (e.g., team=payments)
kubectl cost label --label team=payments --window 30d

# Show idle (wasted) resources per namespace
kubectl cost namespace --show-efficiency --window 30d \
  | awk 'NR>2 && $NF < 0.30 {print "LOW EFFICIENCY: "$0}'

apiVersion: v1
kind: ConfigMap
metadata:
  name: kubecost-alerts
  namespace: kubecost
data:
  alerts.json: |
    {
      "alerts": [
        {
          "type": "budget",
          "threshold": 5000,
          "window": "30d",
          "aggregation": "namespace",
          "filter": "namespace=payments",
          "slackWebhookUrl": "https://hooks.slack.com/YOUR_WEBHOOK",
          "alertNames": ["payments-namespace-budget"]
        },
        {
          "type": "efficiency",
          "efficiencyThreshold": 0.25,
          "window": "7d",
          "aggregation": "deployment",
          "slackWebhookUrl": "https://hooks.slack.com/YOUR_WEBHOOK"
        }
      ]
    }

#!/bin/bash
# Monthly chargeback report — send to finance team
KUBECOST_URL="http://kubecost-cost-analyzer.kubecost:9090"
MONTH=$(date -d "last month" +%Y-%m)
START="${MONTH}-01T00:00:00Z"
END=$(date -d "${MONTH}-01 +1 month -1 second" +%Y-%m-%dT%H:%M:%SZ)

curl -s "${KUBECOST_URL}/model/allocation?window=${START},${END}&aggregate=namespace&accumulate=true" \
  | jq -r '
    .data[0] | to_entries[] |
    {
      namespace: .key,
      team: (.value.properties.labels.team // "untagged"),
      cost_center: (.value.properties.labels."cost-center" // "UNKNOWN"),
      total_usd: (.value.totalCost | . * 100 | round / 100),
      cpu_usd: (.value.cpuCost | . * 100 | round / 100),
      memory_usd: (.value.ramCost | . * 100 | round / 100),
      efficiency_pct: ((.value.totalEfficiency // 0) * 100 | round)
    }' | \
  jq -r '[.namespace, .team, .cost_center, .total_usd, .efficiency_pct] | @csv' \
  > /tmp/chargeback-${MONTH}.csv

echo "Chargeback report: /tmp/chargeback-${MONTH}.csv"
# Upload to S3 for Finance
aws s3 cp /tmp/chargeback-${MONTH}.csv s3://finops-reports/chargeback/

name: Infracost Cost Check

on:
  pull_request:
    paths:
      - '**.tf'
      - '**.tfvars'

jobs:
  infracost:
    name: Estimate infrastructure cost
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      # Generate plan for baseline (base branch)
      - name: Setup Infracost
        uses: infracost/actions/setup@v3
        with:
          api-key: ${{ secrets.INFRACOST_API_KEY }}

      - name: Checkout base branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.base.ref }}
          path: base

      - name: Generate Infracost baseline
        run: |
          infracost breakdown --path=base/terraform \
            --format=json \
            --out-file=/tmp/infracost-base.json

      - name: Generate Infracost for PR
        run: |
          infracost breakdown --path=terraform \
            --format=json \
            --out-file=/tmp/infracost-pr.json

      - name: Generate diff and post PR comment
        run: |
          infracost diff \
            --path=/tmp/infracost-pr.json \
            --compare-to=/tmp/infracost-base.json \
            --format=json \
            --out-file=/tmp/infracost-diff.json

          infracost comment github \
            --path=/tmp/infracost-diff.json \
            --repo=$GITHUB_REPOSITORY \
            --pull-request=${{ github.event.pull_request.number }} \
            --github-token=${{ github.token }} \
            --behavior=update

      - name: Enforce cost policy
        run: |
          infracost comment github \
            --path=/tmp/infracost-diff.json \
            --policy-path=infracost-policy.rego \
            --github-token=${{ github.token }} \
            --repo=$GITHUB_REPOSITORY \
            --pull-request=${{ github.event.pull_request.number }}

version: 0.1

# Custom price overrides for reserved instances
# or private pricing agreements
projects:
  - path: terraform/
    name: prod-eks-cluster
    terraform_vars:
      environment: prod
      region: us-east-1

# Exclude resources from cost check (e.g., tagging-only changes)
exclude_resources:
  - aws_iam_role
  - aws_iam_policy
  - aws_s3_bucket_policy

package infracost

# Block PRs that increase monthly cost by more than $500
deny[msg] {
  monthlyCostDelta := input.diffTotalMonthlyCost
  to_number(monthlyCostDelta) > 500
  msg := sprintf(
    "PR increases monthly cost by $%v (limit: $500). Requires FinOps team approval.",
    [monthlyCostDelta]
  )
}

# Warn on any new On-Demand instance larger than r5.2xlarge
warn[msg] {
  resource := input.projects[_].breakdownDiff.resources[_]
  resource.resourceType == "aws_instance"
  contains(resource.name, "r5.4xlarge")
  resource.monthlyCost > 0
  msg := sprintf(
    "New large On-Demand instance %v may be a candidate for Spot or Reserved Instance.",
    [resource.name]
  )
}

apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: general-spot
spec:
  template:
    metadata:
      labels:
        billing-team: platform
    spec:
      nodeClassRef:
        apiVersion: karpenter.k8s.aws/v1
        kind: EC2NodeClass
        name: default
      requirements:
        # Prefer Spot, fall back to On-Demand automatically
        - key: karpenter.sh/capacity-type
          operator: In
          values: ["spot", "on-demand"]
        # Allow multiple instance families for better Spot availability
        - key: node.kubernetes.io/instance-type
          operator: In
          values:
            - m5.xlarge
            - m5.2xlarge
            - m5a.xlarge
            - m5a.2xlarge
            - m6i.xlarge
            - m6i.2xlarge
            - m6a.xlarge
        - key: kubernetes.io/arch
          operator: In
          values: ["amd64", "arm64"]  # Include Graviton for ~20% savings
        - key: topology.kubernetes.io/zone
          operator: In
          values: ["us-east-1a", "us-east-1b", "us-east-1c"]
  limits:
    cpu: "1000"          # Max 1000 vCPUs in this NodePool
    memory: 4000Gi
  disruption:
    consolidationPolicy: WhenEmptyOrUnderutilized
    consolidateAfter: 60s
    budgets:
      - nodes: "10%"     # Never disrupt more than 10% of nodes at once
---
apiVersion: karpenter.k8s.aws/v1
kind: EC2NodeClass
metadata:
  name: default
spec:
  amiFamily: AL2023
  role: "KarpenterNodeRole-prod"
  subnetSelectorTerms:
    - tags:
        karpenter.sh/discovery: prod-eks-cluster
  securityGroupSelectorTerms:
    - tags:
        karpenter.sh/discovery: prod-eks-cluster
  blockDeviceMappings:
    - deviceName: /dev/xvda
      ebs:
        volumeSize: 100Gi
        volumeType: gp3
        iops: 3000
        throughput: 125
  tags:
    Environment: prod
    ManagedBy: karpenter
    CostCenter: platform

# NodePool for Spark/batch jobs with auto-TTL
# Nodes self-terminate after job completion
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: spark-batch
spec:
  template:
    metadata:
      labels:
        billing-team: data-engineering
        workload-type: batch
    spec:
      nodeClassRef:
        apiVersion: karpenter.k8s.aws/v1
        kind: EC2NodeClass
        name: compute-optimized
      requirements:
        - key: karpenter.sh/capacity-type
          operator: In
          values: ["spot"]  # Batch is always Spot
        - key: node.kubernetes.io/instance-type
          operator: In
          values: ["c5.4xlarge", "c5.8xlarge", "c5a.4xlarge", "c6i.4xlarge"]
      taints:
        - key: workload-type
          value: batch
          effect: NoSchedule
  limits:
    cpu: "500"
  disruption:
    consolidationPolicy: WhenEmpty
    consolidateAfter: 30s    # Aggressive consolidation for batch nodes
    expireAfter: 24h         # Hard TTL — nodes die after 24h max

# Karpenter v1 Disruption Budget — protect critical workloads
# during Spot consolidation
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: general-spot
spec:
  disruption:
    budgets:
      # During business hours (9-5 UTC): very conservative
      - schedule: "0 9 * * mon-fri"
        duration: 8h
        nodes: "5%"
      # Off-hours: aggressive consolidation for savings
      - schedule: "0 17 * * mon-fri"
        duration: 16h
        nodes: "20%"
      # Weekends: most aggressive
      - schedule: "0 0 * * sat"
        duration: 48h
        nodes: "30%"

# Also add PodDisruptionBudget for critical services
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: payments-api-pdb
  namespace: production
spec:
  minAvailable: "80%"
  selector:
    matchLabels:
      app: payments-api

# VPA in Recommendation-only mode (safe for production)
# Never auto-apply without testing Off mode first
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: payments-api-vpa
  namespace: production
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: payments-api
  updatePolicy:
    updateMode: "Off"   # "Off" = recommend only; "Auto" = apply + evict
  resourcePolicy:
    containerPolicies:
      - containerName: "*"
        minAllowed:
          cpu: 50m
          memory: 64Mi
        maxAllowed:
          cpu: 4
          memory: 8Gi
        controlledResources: ["cpu", "memory"]

---
# Query VPA recommendations
# kubectl get vpa -n production -o json | jq '.items[].status.recommendation'

package kubernetes.admission

# OPA Gatekeeper ConstraintTemplate — enforce cost labels on all pods
# Required labels: team, cost-center, environment
deny[msg] {
  input.request.kind.kind == "Pod"
  pod := input.request.object
  required_labels := {"team", "cost-center", "environment"}
  provided := {label | pod.metadata.labels[label]}
  missing := required_labels - provided
  count(missing) > 0
  msg := sprintf(
    "Pod missing required cost labels: %v. All pods must have team, cost-center, environment.",
    [missing]
  )
}

# Block pods without resource limits (will cause OOM and waste)
deny[msg] {
  input.request.kind.kind == "Pod"
  container := input.request.object.spec.containers[_]
  not container.resources.limits
  msg := sprintf(
    "Container '%v' missing resource limits. Set cpu and memory limits to prevent cost overruns.",
    [container.name]
  )
}