From Helm Deployment Configuration to Agent-Generated and Validated Helm Deployment Architecture

# Scaffold a new chart
helm create my-service

# Resulting structure
my-service/
├── Chart.yaml
├── values.yaml
├── charts/
└── templates/
    ├── deployment.yaml
    ├── service.yaml
    ├── ingress.yaml
    ├── hpa.yaml
    └── _helpers.tpl

# templates/deployment.yaml (traditional, hand-authored)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "my-service.fullname" . }}
  labels:
    {{- include "my-service.labels" . | nindent 4 }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      {{- include "my-service.selectorLabels" . | nindent 6 }}
  template:
    metadata:
      labels:
        {{- include "my-service.selectorLabels" . | nindent 8 }}
    spec:
      containers:
        - name: {{ .Chart.Name }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          ports:
            - name: http
              containerPort: {{ .Values.service.port }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}

# Lint the chart
helm lint ./my-service

# Dry-run to catch template errors
helm install my-service ./my-service --dry-run --debug

# Validate rendered YAML against the cluster
helm template my-service ./my-service | kubectl apply --dry-run=client -f -

# Check against OPA/Conftest policies
helm template my-service ./my-service | conftest test -p policies/ -

# .github/workflows/helm-deploy-traditional.yaml
name: Deploy (Traditional)
on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Update kubeconfig
        run: aws eks update-kubeconfig --name my-cluster --region us-east-1

      - name: Helm lint
        run: helm lint ./charts/my-service

      - name: Helm upgrade
        run: |
          helm upgrade --install my-service ./charts/my-service \
            --namespace production \
            --values ./charts/my-service/values-prod.yaml \
            --atomic \
            --timeout 5m          

# intent/my-service.yaml
apiVersion: platform.example.com/v1alpha1
kind: DeploymentIntent
metadata:
  name: my-service
spec:
  image:
    repository: 123456789.dkr.ecr.us-east-1.amazonaws.com/my-service
    tag: v2.3.1
  tier: production          # maps to resource preset and replica policy
  resources:
    profile: medium         # CPU: 500m/1000m, Memory: 512Mi/1Gi
  autoscaling:
    enabled: true
    minReplicas: 2
    maxReplicas: 10
    targetCPUUtilizationPercentage: 70
  rollout:
    strategy: canary
    steps:
      - setWeight: 20
      - pause: {duration: 2m}
      - setWeight: 50
      - pause: {duration: 2m}
      - setWeight: 100
  security:
    runAsNonRoot: true
    readOnlyRootFilesystem: true
    dropAllCapabilities: true
  probe:
    path: /healthz
    port: 8080

# agent/generate_chart.py
"""Chart Generation Agent — reads intent and produces a validated Helm chart."""

import os
import sys
import yaml
import json
from pathlib import Path
from openai import OpenAI

SYSTEM_PROMPT = """
You are a Kubernetes Helm chart expert. Given a DeploymentIntent spec, generate a complete,
production-ready Helm chart. The chart MUST:
- Use Argo Rollouts (rollout.argoproj.io/v1alpha1) instead of Deployment for the workload
- Include a securityContext that enforces runAsNonRoot, readOnlyRootFilesystem, and drops all capabilities
- Include resource requests and limits derived from the resource profile
- Include liveness and readiness probes using the provided probe spec
- Include an HPA if autoscaling is enabled
- Include org-wide mandatory labels: app.kubernetes.io/name, app.kubernetes.io/version, platform.example.com/tier
- Output ONLY valid YAML files separated by --- with a comment indicating the file path

Resource profiles:
  small:  requests cpu=250m mem=256Mi  limits cpu=500m  mem=512Mi
  medium: requests cpu=500m mem=512Mi  limits cpu=1000m mem=1Gi
  large:  requests cpu=1    mem=1Gi    limits cpu=2     mem=2Gi
"""

def load_intent(intent_path: str) -> dict:
    with open(intent_path) as f:
        return yaml.safe_load(f)

def generate_chart(intent: dict, output_dir: Path) -> None:
    client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": f"Generate a Helm chart for this intent:\n\n{yaml.dump(intent)}"},
        ],
        temperature=0.1,
    )

    raw = response.choices[0].message.content
    _write_chart_files(raw, output_dir)
    print(f"✅ Chart generated in {output_dir}")

def _write_chart_files(raw: str, output_dir: Path) -> None:
    """Parse LLM output and write individual chart files."""
    current_path = None
    current_lines: list[str] = []

    for line in raw.splitlines():
        if line.startswith("# charts/") or line.startswith("# Chart.yaml") or line.startswith("# values.yaml"):
            if current_path and current_lines:
                _save(output_dir, current_path, current_lines)
            current_path = line.lstrip("# ").strip()
            current_lines = []
        elif line == "---":
            continue
        else:
            current_lines.append(line)

    if current_path and current_lines:
        _save(output_dir, current_path, current_lines)

def _save(base: Path, rel_path: str, lines: list[str]) -> None:
    target = base / rel_path
    target.parent.mkdir(parents=True, exist_ok=True)
    target.write_text("\n".join(lines))
    print(f"  Written: {target}")

if __name__ == "__main__":
    intent_file = sys.argv[1] if len(sys.argv) > 1 else "intent/my-service.yaml"
    output_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("charts")
    intent = load_intent(intent_file)
    generate_chart(intent, output_dir)

# policies/kyverno-baseline.yaml
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: helm-baseline
  annotations:
    policies.kyverno.io/title: Helm Baseline
    policies.kyverno.io/description: >-
      Enforces security and operational baselines for all Helm-deployed workloads.      
spec:
  validationFailureAction: Enforce
  background: true
  rules:
    - name: require-security-context
      match:
        any:
          - resources:
              kinds: [Deployment, StatefulSet, DaemonSet]
              namespaces: [production, staging]
      validate:
        message: "Containers must set securityContext.runAsNonRoot=true"
        pattern:
          spec:
            template:
              spec:
                containers:
                  - securityContext:
                      runAsNonRoot: true

    - name: require-resource-limits
      match:
        any:
          - resources:
              kinds: [Deployment, StatefulSet, DaemonSet]
      validate:
        message: "All containers must define CPU and memory limits"
        pattern:
          spec:
            template:
              spec:
                containers:
                  - resources:
                      limits:
                        memory: "?*"
                        cpu: "?*"

    - name: require-mandatory-labels
      match:
        any:
          - resources:
              kinds: [Deployment, StatefulSet, DaemonSet, Service]
      validate:
        message: "Resources must carry app.kubernetes.io/name and platform.example.com/tier labels"
        pattern:
          metadata:
            labels:
              app.kubernetes.io/name: "?*"
              platform.example.com/tier: "?*"

    - name: require-liveness-probe
      match:
        any:
          - resources:
              kinds: [Deployment, StatefulSet]
      validate:
        message: "All containers must define a livenessProbe"
        pattern:
          spec:
            template:
              spec:
                containers:
                  - livenessProbe: "?*"

    - name: disallow-privileged-containers
      match:
        any:
          - resources:
              kinds: [Deployment, StatefulSet, DaemonSet]
      validate:
        message: "Privileged containers are not allowed"
        pattern:
          spec:
            template:
              spec:
                containers:
                  - =(securityContext):
                      =(privileged): false

# .github/workflows/helm-ai-deploy.yaml
name: AI-Orchestrated Helm Deploy

on:
  push:
    paths:
      - 'intent/**'
    branches: [main]
  workflow_dispatch:
    inputs:
      intent_file:
        description: 'Path to intent file'
        default: 'intent/my-service.yaml'

permissions:
  id-token: write   # For OIDC authentication to AWS
  contents: read
  issues: write     # Health agent can file issues

jobs:
  # ─────────────────────────────────────────────
  # Stage 1: Generate Helm chart from intent
  # ─────────────────────────────────────────────
  generate:
    name: 🤖 Generate Chart
    runs-on: ubuntu-latest
    outputs:
      chart_path: ${{ steps.gen.outputs.chart_path }}
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install agent dependencies
        run: pip install openai pyyaml

      - name: Generate Helm chart from intent
        id: gen
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          python agent/generate_chart.py \
            ${{ github.event.inputs.intent_file || 'intent/my-service.yaml' }} \
            charts/
          echo "chart_path=charts/my-service" >> $GITHUB_OUTPUT          

      - name: Upload generated chart artifact
        uses: actions/upload-artifact@v4
        with:
          name: generated-chart
          path: charts/

  # ─────────────────────────────────────────────
  # Stage 2: Lint and static validation
  # ─────────────────────────────────────────────
  validate:
    name: 🔍 Validate Chart
    needs: generate
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Download generated chart
        uses: actions/download-artifact@v4
        with:
          name: generated-chart
          path: charts/

      - name: Install Helm
        uses: azure/setup-helm@v4
        with:
          version: '3.14.0'

      - name: Helm lint
        run: helm lint charts/my-service --strict

      - name: Helm template dry-run
        run: |
          helm template my-service charts/my-service \
            --namespace production \
            --debug \
            > /tmp/rendered-manifests.yaml
          echo "✅ Template rendered successfully"          

      - name: Install Checkov
        run: pip install checkov

      - name: Checkov misconfiguration scan
        run: |
          checkov -f /tmp/rendered-manifests.yaml \
            --framework kubernetes \
            --compact \
            --quiet \
            --soft-fail-on MEDIUM          

      - name: Install Kyverno CLI
        run: |
          curl -LO https://github.com/kyverno/kyverno/releases/download/v1.12.3/kyverno-cli_v1.12.3_linux_x86_64.tar.gz
          tar -xzf kyverno-cli_v1.12.3_linux_x86_64.tar.gz
          sudo mv kyverno /usr/local/bin/
          kyverno version          

      - name: Kyverno policy validation
        run: |
          kyverno apply policies/kyverno-baseline.yaml \
            --resource /tmp/rendered-manifests.yaml \
            --detailed-results          

      - name: Upload validated manifests
        uses: actions/upload-artifact@v4
        with:
          name: validated-manifests
          path: /tmp/rendered-manifests.yaml

  # ─────────────────────────────────────────────
  # Stage 3: Progressive deployment to EKS
  # ─────────────────────────────────────────────
  deploy:
    name: 🚀 Deploy to EKS
    needs: validate
    runs-on: ubuntu-latest
    environment: production
    steps:
      - uses: actions/checkout@v4

      - name: Download generated chart
        uses: actions/download-artifact@v4
        with:
          name: generated-chart
          path: charts/

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Update kubeconfig
        run: aws eks update-kubeconfig --name my-cluster --region us-east-1

      - name: Install Helm
        uses: azure/setup-helm@v4
        with:
          version: '3.14.0'

      - name: Install Argo Rollouts kubectl plugin
        run: |
          curl -LO https://github.com/argoproj/argo-rollouts/releases/latest/download/kubectl-argo-rollouts-linux-amd64
          chmod +x kubectl-argo-rollouts-linux-amd64
          sudo mv kubectl-argo-rollouts-linux-amd64 /usr/local/bin/kubectl-argo-rollouts          

      - name: Helm upgrade (canary rollout)
        id: helm_upgrade
        run: |
          helm upgrade --install my-service charts/my-service \
            --namespace production \
            --create-namespace \
            --set rollout.enabled=true \
            --atomic \
            --timeout 2m \
            --wait          

      - name: Watch Argo Rollout progress
        run: |
          echo "Watching rollout progression..."
          kubectl argo rollouts status my-service -n production --timeout 10m          

      - name: Verify rollout health
        run: |
          STATUS=$(kubectl argo rollouts get rollout my-service -n production -o json \
            | jq -r '.status.phase')
          echo "Rollout status: $STATUS"
          if [ "$STATUS" != "Healthy" ]; then
            echo "❌ Rollout not healthy. Initiating rollback..."
            kubectl argo rollouts undo my-service -n production
            exit 1
          fi
          echo "✅ Rollout healthy"          

  # ─────────────────────────────────────────────
  # Stage 4: Post-deploy health monitoring
  # ─────────────────────────────────────────────
  monitor:
    name: 📊 Health Monitor
    needs: deploy
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install monitoring dependencies
        run: pip install openai pyyaml kubernetes requests

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Update kubeconfig
        run: aws eks update-kubeconfig --name my-cluster --region us-east-1

      - name: Run health monitoring agent
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_REPOSITORY: ${{ github.repository }}
        run: |
          python agent/health_monitor.py \
            --namespace production \
            --deployment my-service \
            --duration 300          

# agent/health_monitor.py
"""
Continuous Health Agent — monitors a deployment post-rollout, detects anomalies,
and files GitHub Issues when intervention is required.
"""

import argparse
import json
import os
import sys
import time
from datetime import datetime, timezone
from typing import Any

import requests
from kubernetes import client, config
from openai import OpenAI


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument("--namespace", default="production")
    parser.add_argument("--deployment", required=True)
    parser.add_argument("--duration", type=int, default=300, help="Monitoring window in seconds")
    return parser.parse_args()


def collect_pod_events(v1: client.CoreV1Api, namespace: str, deployment: str) -> list[dict]:
    pods = v1.list_namespaced_pod(namespace, label_selector=f"app.kubernetes.io/name={deployment}")
    events: list[dict] = []
    for pod in pods.items:
        pod_events = v1.list_namespaced_event(
            namespace, field_selector=f"involvedObject.name={pod.metadata.name}"
        )
        for ev in pod_events.items:
            events.append({
                "pod": pod.metadata.name,
                "type": ev.type,
                "reason": ev.reason,
                "message": ev.message,
                "count": ev.count,
            })
        # Check for OOMKill / CrashLoopBackOff
        if pod.status and pod.status.container_statuses:
            for cs in pod.status.container_statuses:
                if cs.state and cs.state.waiting:
                    if cs.state.waiting.reason in ("CrashLoopBackOff", "OOMKilled"):
                        events.append({
                            "pod": pod.metadata.name,
                            "type": "Warning",
                            "reason": cs.state.waiting.reason,
                            "message": cs.state.waiting.message or cs.state.waiting.reason,
                            "count": cs.restart_count,
                        })
    return events


def analyze_health(events: list[dict], deployment: str) -> dict[str, Any]:
    """Use LLM to analyze events and recommend action."""
    openai_client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])

    prompt = f"""
You are a Kubernetes SRE agent. Analyze the following events for deployment '{deployment}' 
and determine:
1. Is the deployment healthy? (yes/no)
2. What is the severity? (ok / warning / critical)
3. What is the root cause if unhealthy?
4. What remediation action should be taken? (none / scale_down / rollback / resource_increase / investigate)
5. A brief human-readable summary.

Events:
{json.dumps(events, indent=2)}

Respond with valid JSON only:
{{
  "healthy": bool,
  "severity": "ok|warning|critical",
  "root_cause": "string",
  "action": "none|scale_down|rollback|resource_increase|investigate",
  "summary": "string"
}}
"""
    response = openai_client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": prompt}],
        response_format={"type": "json_object"},
        temperature=0,
    )
    return json.loads(response.choices[0].message.content)


def file_github_issue(deployment: str, analysis: dict[str, Any]) -> None:
    """File a GitHub Issue when the agent cannot self-remediate."""
    token = os.environ.get("GITHUB_TOKEN")
    repo = os.environ.get("GITHUB_REPOSITORY")
    if not token or not repo:
        print("⚠️  GITHUB_TOKEN or GITHUB_REPOSITORY not set — skipping issue creation")
        return

    title = f"[Health Agent] {deployment}: {analysis['severity'].upper()} — {analysis['root_cause']}"
    body = f"""## Deployment Health Alert

**Deployment:** `{deployment}`  
**Severity:** {analysis['severity']}  
**Detected at:** {datetime.now(timezone.utc).isoformat()}

### Root Cause
{analysis['root_cause']}

### Summary
{analysis['summary']}

### Recommended Action
`{analysis['action']}`

> This issue was automatically created by the Continuous Health Agent.
"""
    resp = requests.post(
        f"https://api.github.com/repos/{repo}/issues",
        headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"},
        json={"title": title, "body": body, "labels": ["health-agent", "deployment"]},
        timeout=30,
    )
    if resp.status_code == 201:
        print(f"📋 GitHub Issue created: {resp.json()['html_url']}")
    else:
        print(f"⚠️  Failed to create issue: {resp.status_code} {resp.text}")


def main() -> None:
    args = parse_args()
    config.load_kube_config()
    v1 = client.CoreV1Api()

    print(f"🔍 Monitoring deployment '{args.deployment}' in '{args.namespace}' for {args.duration}s...")
    deadline = time.time() + args.duration

    while time.time() < deadline:
        events = collect_pod_events(v1, args.namespace, args.deployment)

        if events:
            analysis = analyze_health(events, args.deployment)
            print(f"\n[{datetime.now(timezone.utc).isoformat()}] Health analysis:")
            print(json.dumps(analysis, indent=2))

            if analysis["severity"] == "critical":
                print(f"🚨 Critical issue detected. Action: {analysis['action']}")
                file_github_issue(args.deployment, analysis)
                if analysis["action"] == "rollback":
                    print("⏪ Initiating automated rollback via kubectl argo rollouts...")
                    os.system(f"kubectl argo rollouts undo {args.deployment} -n {args.namespace}")
                sys.exit(1)
            elif analysis["severity"] == "warning":
                print(f"⚠️  Warning: {analysis['summary']}")
        else:
            print(f"✅ [{datetime.now(timezone.utc).isoformat()}] No anomalous events detected")

        time.sleep(30)

    print(f"\n✅ Monitoring complete — '{args.deployment}' is healthy.")


if __name__ == "__main__":
    main()

# charts/my-service/templates/rollout.yaml  (AI-generated)
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: {{ include "my-service.fullname" . }}
  labels:
    {{- include "my-service.labels" . | nindent 4 }}
spec:
  replicas: {{ .Values.replicaCount }}
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      {{- include "my-service.selectorLabels" . | nindent 6 }}
  template:
    metadata:
      labels:
        {{- include "my-service.selectorLabels" . | nindent 8 }}
    spec:
      securityContext:
        runAsNonRoot: true
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: {{ .Chart.Name }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: [ALL]
          ports:
            - name: http
              containerPort: {{ .Values.service.port }}
              protocol: TCP
          livenessProbe:
            httpGet:
              path: {{ .Values.probe.path }}
              port: {{ .Values.probe.port }}
            initialDelaySeconds: 10
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: {{ .Values.probe.path }}
              port: {{ .Values.probe.port }}
            initialDelaySeconds: 5
            periodSeconds: 5
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
  strategy:
    canary:
      steps:
        {{- toYaml .Values.rollout.steps | nindent 8 }}
      analysis:
        templates:
          - templateName: success-rate
        startingStep: 1
        args:
          - name: service-name
            value: {{ include "my-service.fullname" . }}
---
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate
spec:
  args:
    - name: service-name
  metrics:
    - name: success-rate
      interval: 1m
      successCondition: result[0] >= 0.95
      failureLimit: 3
      provider:
        prometheus:
          address: http://prometheus-operated.monitoring:9090
          query: |
            sum(rate(http_requests_total{
              job="{{ args.service-name }}",
              status=~"2.."
            }[2m]))
            /
            sum(rate(http_requests_total{
              job="{{ args.service-name }}"
            }[2m]))