Skip to main content

Python for Java Developers: Cloud-Native Data & AI Platform Engineering - IX

· 21 min read

Based on our progression and the current market landscape, the best next chapter would be:

🎯 Chapater 9: Cloud-Native Data & AI Platform Engineering​

Why This is the Perfect Next Step:​

  1. Natural Progression: You have MLOps → Now need cloud infrastructure to run it at scale
  2. Industry Shift: Everything is moving to cloud-native architectures
  3. Java Developer Advantage: You understand distributed systems from Java world
  4. Career Peak: Cloud Data/AI Architects are top-tier in compensation and demand

📊 Chapater 5.4: Building Cloud-Native Data & AI Platforms​

The Evolution You're Completing:​

Java Developer → Data Engineer → MLOps Engineer → Cloud Data/AI Architect
Local Code → Spark Jobs → ML Pipelines → Cloud-Native Platforms
Spring Boot → Airflow DAGs → FastAPI Models → Kubernetes, Terraform, Cloud Services

🚀 Chapter Outline: Cloud-Native Data/AI Platforms​

1. Cloud Data Platforms: AWS vs Azure vs GCP​

# Compare: On-prem Hadoop vs Cloud Data Platforms
# Java/Hadoop: HDFS, YARN, MapReduce, Hive
# Cloud: S3, Databricks, Snowflake, BigQuery, Redshift

# Cloud-agnostic platform design
from abc import ABC, abstractmethod
from typing import Dict, Any
import boto3
from google.cloud import bigquery
import snowflake.connector

class CloudDataPlatform(ABC):
"""Abstract cloud data platform (strategy pattern)"""

@abstractmethod
def store_data(self, data, path: str):
"""Store data in cloud storage"""
pass

@abstractmethod
def process_data(self, query: str) -> Any:
"""Process data using cloud engine"""
pass

@abstractmethod
def serve_data(self, query: str, format: str = "dataframe"):
"""Serve query results"""
pass

class AWSDataPlatform(CloudDataPlatform):
"""AWS data platform implementation"""

def __init__(self):
self.s3 = boto3.client('s3')
self.athena = boto3.client('athena')
self.glue = boto3.client('glue')

def store_data(self, data, path: str):
"""Store in S3"""
bucket, key = self._parse_s3_path(path)
self.s3.put_object(Bucket=bucket, Key=key, Body=data)

def process_data(self, query: str):
"""Process with Athena/Glue"""
# Catalog with Glue
self.glue.create_table_if_not_exists(...)

# Query with Athena
response = self.athena.start_query_execution(
QueryString=query,
ResultConfiguration={'OutputLocation': 's3://query-results/'}
)
return response['QueryExecutionId']

def serve_data(self, query: str, format: str = "dataframe"):
"""Serve via Redshift/Athena"""
import pandas as pd

if format == "dataframe":
# Use pandas with S3 select
df = pd.read_parquet(f"s3://data-lake/{query}.parquet")
return df
else:
# Direct SQL endpoint
return self.athena.get_query_results(...)

class SnowflakeDataPlatform(CloudDataPlatform):
"""Snowflake implementation"""

def __init__(self):
self.conn = snowflake.connector.connect(
user=os.getenv('SNOWFLAKE_USER'),
password=os.getenv('SNOWFLAKE_PASSWORD'),
account=os.getenv('SNOWFLAKE_ACCOUNT'),
warehouse='COMPUTE_WH',
database='PRODUCTION',
schema='PUBLIC'
)

def process_data(self, query: str):
"""Process in Snowflake"""
cursor = self.conn.cursor()
cursor.execute(query)
return cursor.fetchall()

def store_data(self, data, stage: str):
"""Stage data in Snowflake"""
cursor = self.conn.cursor()

# Create stage if not exists
cursor.execute(f"CREATE STAGE IF NOT EXISTS {stage}")

# Upload to stage
# ... upload logic

def serve_data(self, query: str, format: str = "dataframe"):
"""Serve via Snowflake"""
import pandas as pd

cursor = self.conn.cursor()
cursor.execute(query)

if format == "dataframe":
df = cursor.fetch_pandas_all()
return df
else:
return cursor.fetchall()

# Factory pattern for cloud platform selection
class DataPlatformFactory:
"""Factory to create appropriate data platform"""

@staticmethod
def create(cloud: str, config: Dict) -> CloudDataPlatform:
platforms = {
'aws': AWSDataPlatform,
'gcp': lambda: GoogleDataPlatform(config),
'azure': lambda: AzureDataPlatform(config),
'snowflake': SnowflakeDataPlatform,
'databricks': DatabricksDataPlatform
}

platform_class = platforms.get(cloud.lower())
if not platform_class:
raise ValueError(f"Unsupported cloud: {cloud}")

return platform_class()

2. Infrastructure as Code: Terraform & Pulumi​

# Compare: Java deployment (WAR/JAR) vs Infrastructure as Code
# Java: Maven/Gradle builds, manual server provisioning
# Python: Terraform/Pulumi, automated infrastructure

import pulumi
import pulumi_aws as aws
import pulumi_docker as docker
import pulumi_kubernetes as k8s
from typing import Dict

class DataPlatformInfrastructure:
"""Define complete data platform infrastructure"""

def __init__(self, env: str = "prod"):
self.env = env
self.config = self._load_config()

def _load_config(self) -> Dict:
"""Load configuration based on environment"""
configs = {
'dev': {
'instance_type': 't3.medium',
'node_count': 3,
'storage_gb': 100
},
'prod': {
'instance_type': 'r5.2xlarge',
'node_count': 10,
'storage_gb': 1000
}
}
return configs.get(self.env, configs['dev'])

def create_vpc(self):
"""Create VPC for data platform"""
vpc = aws.ec2.Vpc(
f"data-platform-vpc-{self.env}",
cidr_block="10.0.0.0/16",
enable_dns_hostnames=True,
enable_dns_support=True,
tags={"Environment": self.env, "Type": "DataPlatform"}
)

# Subnets
public_subnet = aws.ec2.Subnet(
f"public-subnet-{self.env}",
vpc_id=vpc.id,
cidr_block="10.0.1.0/24",
map_public_ip_on_launch=True,
availability_zone="us-east-1a"
)

private_subnet = aws.ec2.Subnet(
f"private-subnet-{self.env}",
vpc_id=vpc.id,
cidr_block="10.0.2.0/24",
availability_zone="us-east-1b"
)

return vpc, public_subnet, private_subnet

def create_eks_cluster(self, vpc, subnet):
"""Create EKS cluster for running data workloads"""
cluster = aws.eks.Cluster(
f"data-platform-eks-{self.env}",
role_arn=aws.iam.Role(
f"eks-role-{self.env}",
assume_role_policy="""{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {"Service": "eks.amazonaws.com"},
"Action": "sts:AssumeRole"
}]
}"""
).arn,
vpc_config=aws.eks.ClusterVpcConfigArgs(
subnet_ids=[subnet.id],
endpoint_private_access=True,
endpoint_public_access=True
),
tags={"Environment": self.env}
)

# Node group
node_group = aws.eks.NodeGroup(
f"data-node-group-{self.env}",
cluster_name=cluster.name,
node_role_arn=aws.iam.Role(
f"node-role-{self.env}",
assume_role_policy="""{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {"Service": "ec2.amazonaws.com"},
"Action": "sts:AssumeRole"
}]
}"""
).arn,
subnet_ids=[subnet.id],
scaling_config=aws.eks.NodeGroupScalingConfigArgs(
desired_size=self.config['node_count'],
max_size=self.config['node_count'] * 2,
min_size=1
),
instance_types=[self.config['instance_type']],
disk_size=self.config['storage_gb']
)

return cluster, node_group

def create_data_services(self, cluster):
"""Deploy data services to EKS"""

# Spark Operator
spark_operator = k8s.helm.v3.Release(
"spark-operator",
chart="spark-operator",
repository_opts=k8s.helm.v3.RepositoryOptsArgs(
repo="https://googlecloudplatform.github.io/spark-on-k8s-operator"
),
namespace="spark-operator",
values={
"serviceAccounts": {
"spark": {"name": "spark"}
}
}
)

# Airflow
airflow = k8s.helm.v3.Release(
"airflow",
chart="airflow",
repository_opts=k8s.helm.v3.RepositoryOptsArgs(
repo="https://airflow.apache.org"
),
namespace="airflow",
values={
"executor": "KubernetesExecutor",
"redis": {"enabled": True},
"postgresql": {"enabled": True},
"workers": {
"replicas": 3,
"resources": {
"requests": {"memory": "2Gi", "cpu": "500m"},
"limits": {"memory": "4Gi", "cpu": "1000m"}
}
}
}
)

# MLflow
mlflow = k8s.helm.v3.Release(
"mlflow",
chart="mlflow",
repository_opts=k8s.helm.v3.RepositoryOptsArgs(
repo="https://community-charts.github.io/helm-charts"
),
namespace="mlflow",
values={
"backendStore": {
"postgresql": {
"enabled": True,
"database": "mlflow"
}
},
"defaultArtifactRoot": "s3://mlflow-artifacts/",
"service": {
"type": "LoadBalancer",
"port": 5000
}
}
)

return {
"spark_operator": spark_operator,
"airflow": airflow,
"mlflow": mlflow
}

def create_monitoring_stack(self):
"""Create monitoring stack (Prometheus, Grafana)"""
monitoring_namespace = k8s.core.v1.Namespace(
"monitoring",
metadata={"name": "monitoring"}
)

# Prometheus
prometheus = k8s.helm.v3.Release(
"prometheus",
chart="prometheus",
repository_opts=k8s.helm.v3.RepositoryOptsArgs(
repo="https://prometheus-community.github.io/helm-charts"
),
namespace="monitoring",
values={
"alertmanager": {"enabled": True},
"pushgateway": {"enabled": True},
"nodeExporter": {"enabled": True},
"server": {
"persistentVolume": {
"enabled": True,
"size": "50Gi"
}
}
}
)

# Grafana
grafana = k8s.helm.v3.Release(
"grafana",
chart="grafana",
repository_opts=k8s.helm.v3.RepositoryOptsArgs(
repo="https://grafana.github.io/helm-charts"
),
namespace="monitoring",
values={
"adminPassword": pulumi.Output.secret("admin123"),
"persistence": {
"enabled": True,
"size": "10Gi"
},
"datasources": {
"datasources.yaml": {
"apiVersion": 1,
"datasources": [{
"name": "Prometheus",
"type": "prometheus",
"url": "http://prometheus-server.monitoring.svc.cluster.local",
"access": "proxy"
}]
}
},
"dashboardProviders": {
"dashboardproviders.yaml": {
"apiVersion": 1,
"providers": [{
"name": "default",
"orgId": 1,
"folder": "",
"type": "file",
"disableDeletion": False,
"editable": True,
"options": {"path": "/var/lib/grafana/dashboards"}
}]
}
}
}
)

return prometheus, grafana

# Deploy entire platform
platform = DataPlatformInfrastructure(env="prod")
vpc, public_subnet, private_subnet = platform.create_vpc()
cluster, node_group = platform.create_eks_cluster(vpc, private_subnet)
data_services = platform.create_data_services(cluster)
prometheus, grafana = platform.create_monitoring_stack()

# Export outputs
pulumi.export("vpc_id", vpc.id)
pulumi.export("cluster_name", cluster.name)
pulumi.export("airflow_url", pulumi.Output.concat("http://", data_services["airflow"].status.load_balancer.ingress[0].hostname))
pulumi.export("mlflow_url", pulumi.Output.concat("http://", data_services["mlflow"].status.load_balancer.ingress[0].hostname, ":5000"))
pulumi.export("grafana_url", pulumi.Output.concat("http://", grafana.status.load_balancer.ingress[0].hostname))

3. Kubernetes-native Data Processing​

# Compare: YARN (Hadoop) vs Kubernetes for data processing
# Java/Hadoop: YARN resource management, HDFS storage
# Kubernetes-native: Spark on K8s, Dask on K8s, Ray on K8s

from kubernetes import client, config
import yaml
from typing import Dict, Any
import json

class KubernetesDataPlatform:
"""Kubernetes-native data processing platform"""

def __init__(self):
config.load_kube_config()
self.api = client.CoreV1Api()
self.batch_api = client.BatchV1Api()
self.custom_api = client.CustomObjectsApi()

def submit_spark_job(self, job_spec: Dict):
"""Submit Spark job to Kubernetes"""
# Create SparkApplication CRD
spark_app = {
"apiVersion": "sparkoperator.k8s.io/v1beta2",
"kind": "SparkApplication",
"metadata": {
"name": job_spec["name"],
"namespace": "spark-jobs"
},
"spec": {
"type": "Python",
"pythonVersion": "3",
"mode": "cluster",
"image": job_spec.get("image", "bitnami/spark:3.4"),
"imagePullPolicy": "Always",
"mainApplicationFile": job_spec["main_file"],
"sparkVersion": "3.4.0",
"restartPolicy": {
"type": "OnFailure",
"onFailureRetries": 3,
"onFailureRetryInterval": 10
},
"driver": {
"cores": job_spec.get("driver_cores", 1),
"memory": job_spec.get("driver_memory", "2g"),
"serviceAccount": "spark",
"env": job_spec.get("env", []),
"labels": {"version": "3.4.0"}
},
"executor": {
"cores": job_spec.get("executor_cores", 2),
"instances": job_spec.get("executor_instances", 3),
"memory": job_spec.get("executor_memory", "4g"),
"env": job_spec.get("env", []),
"labels": {"version": "3.4.0"}
},
"arguments": job_spec.get("arguments", [])
}
}

# Submit to Kubernetes
self.custom_api.create_namespaced_custom_object(
group="sparkoperator.k8s.io",
version="v1beta2",
namespace="spark-jobs",
plural="sparkapplications",
body=spark_app
)

return spark_app["metadata"]["name"]

def submit_dask_job(self, job_spec: Dict):
"""Submit Dask job to Kubernetes"""
# Dask scheduler
scheduler_spec = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": f"{job_spec['name']}-scheduler",
"namespace": "dask-jobs"
},
"spec": {
"replicas": 1,
"selector": {"matchLabels": {"app": f"{job_spec['name']}-scheduler"}},
"template": {
"metadata": {"labels": {"app": f"{job_spec['name']}-scheduler"}},
"spec": {
"containers": [{
"name": "scheduler",
"image": "daskdev/dask:latest",
"command": ["dask-scheduler"],
"ports": [{"containerPort": 8786}],
"resources": {
"requests": {"memory": "1Gi", "cpu": "200m"},
"limits": {"memory": "2Gi", "cpu": "500m"}
}
}]
}
}
}
}

# Dask workers
worker_spec = {
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": f"{job_spec['name']}-worker",
"namespace": "dask-jobs"
},
"spec": {
"replicas": job_spec.get("worker_count", 3),
"selector": {"matchLabels": {"app": f"{job_spec['name']}-worker"}},
"template": {
"metadata": {"labels": {"app": f"{job_spec['name']}-worker"}},
"spec": {
"containers": [{
"name": "worker",
"image": "daskdev/dask:latest",
"command": [
"dask-worker",
f"{job_spec['name']}-scheduler:8786"
],
"resources": {
"requests": {"memory": "4Gi", "cpu": "1000m"},
"limits": {"memory": "8Gi", "cpu": "2000m"}
}
}]
}
}
}
}

# Create deployments
apps_api = client.AppsV1Api()
apps_api.create_namespaced_deployment(
namespace="dask-jobs",
body=scheduler_spec
)

apps_api.create_namespaced_deployment(
namespace="dask-jobs",
body=worker_spec
)

# Create service for scheduler
service_spec = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": f"{job_spec['name']}-scheduler",
"namespace": "dask-jobs"
},
"spec": {
"selector": {"app": f"{job_spec['name']}-scheduler"},
"ports": [{"port": 8786, "targetPort": 8786}],
"type": "ClusterIP"
}
}

self.api.create_namespaced_service(
namespace="dask-jobs",
body=service_spec
)

def submit_ray_job(self, job_spec: Dict):
"""Submit Ray job to Kubernetes"""
# Ray cluster
ray_cluster = {
"apiVersion": "ray.io/v1alpha1",
"kind": "RayCluster",
"metadata": {
"name": job_spec["name"],
"namespace": "ray-jobs"
},
"spec": {
"headGroupSpec": {
"serviceType": "ClusterIP",
"rayStartParams": {
"dashboard-host": "0.0.0.0",
"num-cpus": str(job_spec.get("head_cpus", 4))
},
"template": {
"spec": {
"containers": [{
"name": "ray-head",
"image": "rayproject/ray:2.8.0",
"ports": [
{"containerPort": 6379, "name": "gcs"},
{"containerPort": 8265, "name": "dashboard"},
{"containerPort": 10001, "name": "client"}
],
"lifecycle": {
"preStop": {
"exec": {
"command": ["/bin/sh", "-c", "ray stop"]
}
}
},
"resources": {
"limits": {
"cpu": str(job_spec.get("head_cpus", 4)),
"memory": job_spec.get("head_memory", "8Gi")
}
}
}]
}
}
},
"workerGroupSpecs": [{
"replicas": job_spec.get("worker_count", 3),
"minReplicas": 1,
"maxReplicas": 10,
"groupName": "small-worker-group",
"rayStartParams": {"num-cpus": "2"},
"template": {
"spec": {
"containers": [{
"name": "ray-worker",
"image": "rayproject/ray:2.8.0",
"resources": {
"limits": {
"cpu": "2",
"memory": "4Gi"
}
}
}]
}
}
}]
}
}

# Submit RayCluster CRD
self.custom_api.create_namespaced_custom_object(
group="ray.io",
version="v1alpha1",
namespace="ray-jobs",
plural="rayclusters",
body=ray_cluster
)

# Submit Ray job
ray_job = {
"apiVersion": "ray.io/v1alpha1",
"kind": "RayJob",
"metadata": {
"name": f"{job_spec['name']}-job",
"namespace": "ray-jobs"
},
"spec": {
"entrypoint": job_spec["entrypoint"],
"runtimeEnv": job_spec.get("runtime_env", ""),
"clusterSelector": {
"ray.io/cluster": job_spec["name"]
},
"submitterPodTemplate": {
"spec": {
"containers": [{
"name": "ray-job-submitter",
"image": "rayproject/ray:2.8.0",
"resources": {
"requests": {"cpu": "100m", "memory": "256Mi"}
}
}]
}
}
}
}

self.custom_api.create_namespaced_custom_object(
group="ray.io",
version="v1alpha1",
namespace="ray-jobs",
plural="rayjobs",
body=ray_job
)

def auto_scale_based_on_metrics(self):
"""Autoscale based on metrics (like YARN but better)"""
from kubernetes.client import V2beta2HorizontalPodAutoscaler

# Create HPA for Spark executors
hpa = V2beta2HorizontalPodAutoscaler(
api_version="autoscaling/v2beta2",
kind="HorizontalPodAutoscaler",
metadata=client.V1ObjectMeta(
name="spark-executor-autoscaler",
namespace="spark-jobs"
),
spec={
"scaleTargetRef": {
"apiVersion": "sparkoperator.k8s.io/v1beta2",
"kind": "SparkApplication",
"name": "data-processing-job"
},
"minReplicas": 2,
"maxReplicas": 20,
"metrics": [
{
"type": "Resource",
"resource": {
"name": "cpu",
"target": {
"type": "Utilization",
"averageUtilization": 70
}
}
},
{
"type": "Resource",
"resource": {
"name": "memory",
"target": {
"type": "Utilization",
"averageUtilization": 80
}
}
},
{
"type": "Pods",
"pods": {
"metric": {
"name": "spark_queue_length"
},
"target": {
"type": "AverageValue",
"averageValue": "10"
}
}
}
]
}
)

autoscaling_api = client.AutoscalingV2beta2Api()
autoscaling_api.create_namespaced_horizontal_pod_autoscaler(
namespace="spark-jobs",
body=hpa
)

4. Serverless Data Processing​

# Compare: Always-on clusters vs Serverless
# Traditional: Always-running Spark clusters ($$$)
# Serverless: AWS Lambda, Google Cloud Run, Azure Functions

import aws_cdk as cdk
from aws_cdk import (
aws_lambda as lambda_,
aws_stepfunctions as sfn,
aws_stepfunctions_tasks as tasks,
aws_s3 as s3,
aws_events as events,
aws_events_targets as targets,
Duration
)

class ServerlessDataPlatform(cdk.Stack):
"""Serverless data platform on AWS"""

def __init__(self, scope: cdk.App, id: str, **kwargs):
super().__init__(scope, id, **kwargs)

# S3 data lake
data_lake = s3.Bucket(
self, "DataLake",
versioned=True,
encryption=s3.BucketEncryption.S3_MANAGED,
lifecycle_rules=[
s3.LifecycleRule(
transitions=[
s3.Transition(
storage_class=s3.StorageClass.INFREQUENT_ACCESS,
transition_after=Duration.days(30)
),
s3.Transition(
storage_class=s3.StorageClass.GLACIER,
transition_after=Duration.days(90)
)
]
)
]
)

# Lambda functions for data processing
process_lambda = lambda_.Function(
self, "ProcessData",
runtime=lambda_.Runtime.PYTHON_3_11,
code=lambda_.Code.from_asset("lambda"),
handler="process.handler",
timeout=Duration.minutes(15),
memory_size=10240, # 10GB max for Lambda
environment={
"DATA_LAKE_BUCKET": data_lake.bucket_name,
"PYSPARK_SUBMIT_ARGS": "--conf spark.driver.memory=5g --conf spark.executor.memory=5g pyspark-shell"
},
layers=[
lambda_.LayerVersion.from_layer_version_arn(
self, "PySparkLayer",
"arn:aws:lambda:us-east-1:123456789012:layer:pyspark-layer:1"
)
]
)

data_lake.grant_read_write(process_lambda)

# Step Functions for workflow orchestration
process_task = tasks.LambdaInvoke(
self, "ProcessDataTask",
lambda_function=process_lambda,
output_path="$.Payload"
)

validate_task = tasks.LambdaInvoke(
self, "ValidateDataTask",
lambda_function=lambda_.Function(
self, "ValidateData",
runtime=lambda_.Runtime.PYTHON_3_11,
code=lambda_.Code.from_asset("lambda/validate"),
handler="validate.handler"
)
)

notify_task = tasks.LambdaInvoke(
self, "NotifyTask",
lambda_function=lambda_.Function(
self, "Notify",
runtime=lambda_.Runtime.PYTHON_3_11,
code=lambda_.Code.from_asset("lambda/notify"),
handler="notify.handler"
)
)

# Define workflow
definition = process_task.next(
validate_task
).next(
sfn.Choice(self, "ValidationPassed?")
.when(
sfn.Condition.boolean_equals("$.validation.passed", True),
notify_task.add_catch(
notify_task, errors=["States.ALL"], result_path="$.error"
)
)
.otherwise(
tasks.LambdaInvoke(
self, "HandleFailure",
lambda_function=lambda_.Function(
self, "HandleFailure",
runtime=lambda_.Runtime.PYTHON_3_11,
code=lambda_.Code.from_asset("lambda/failure"),
handler="failure.handler"
)
)
)
)

state_machine = sfn.StateMachine(
self, "DataPipelineStateMachine",
definition=definition,
timeout=Duration.hours(2)
)

# Schedule execution
rule = events.Rule(
self, "DailySchedule",
schedule=events.Schedule.cron(
minute="0",
hour="2"
)
)

rule.add_target(targets.SfnStateMachine(state_machine))

# API Gateway for ad-hoc processing
from aws_cdk import aws_apigateway as apigateway

api = apigateway.RestApi(
self, "DataProcessingAPI",
rest_api_name="Data Processing Service",
description="Serverless data processing API"
)

process_resource = api.root.add_resource("process")
process_resource.add_method(
"POST",
apigateway.LambdaIntegration(process_lambda),
authorization_type=apigateway.AuthorizationType.IAM
)

# Lambda function code (process.py)
"""
import json
import boto3
import pandas as pd
from pyspark.sql import SparkSession
from io import StringIO

def handler(event, context):
# Initialize Spark session
spark = SparkSession.builder \
.appName("LambdaPySpark") \
.config("spark.sql.shuffle.partitions", "2") \
.getOrCreate()

# Read from S3
s3 = boto3.client('s3')
bucket = event['bucket']
key = event['key']

obj = s3.get_object(Bucket=bucket, Key=key)
data = obj['Body'].read().decode('utf-8')

# Process with Pandas (small data) or PySpark (large data)
if len(data) < 1000000: # < 1MB
df = pd.read_csv(StringIO(data))
# Process with Pandas
result = df.groupby('category').sum().to_dict()
else:
# Process with PySpark
df = spark.read.csv(StringIO(data), header=True, inferSchema=True)
result = df.groupBy('category').sum().collect()
result = {row['category']: row['sum(amount)'] for row in result}

# Write result back to S3
output_key = f"processed/{key}"
s3.put_object(
Bucket=bucket,
Key=output_key,
Body=json.dumps(result)
)

return {
'statusCode': 200,
'body': json.dumps({
'processed': True,
'output_location': f"s3://{bucket}/{output_key}"
})
}
"""

5. Data Mesh & Domain-Oriented Architecture​

# Compare: Centralized Data Warehouse vs Data Mesh
# Traditional: Central ETL, single data team, monolithic warehouse
# Data Mesh: Domain-oriented, distributed ownership, federated governance

from typing import Dict, List, Optional
from dataclasses import dataclass
from enum import Enum
import json
import yaml

class DataProductType(Enum):
"""Types of data products in data mesh"""
SOURCE_ALIGNED = "source_aligned"
AGGREGATED = "aggregated"
CONSUMER_ALIGNED = "consumer_aligned"

@dataclass
class DataProduct:
"""Data product definition"""
name: str
domain: str
type: DataProductType
owner: str
sla: str # Service Level Agreement
schema: Dict
quality_metrics: Dict
access_policies: List[Dict]

def to_manifest(self) -> Dict:
"""Generate data product manifest"""
return {
"apiVersion": "datamesh.io/v1alpha1",
"kind": "DataProduct",
"metadata": {
"name": self.name,
"domain": self.domain,
"labels": {
"owner": self.owner,
"type": self.type.value
}
},
"spec": {
"schema": self.schema,
"sla": self.sla,
"qualityMetrics": self.quality_metrics,
"accessPolicies": self.access_policies,
"ports": [
{
"name": "analytical",
"protocol": "http",
"service": f"{self.name}-analytical",
"dataFormat": "parquet"
},
{
"name": "operational",
"protocol": "http",
"service": f"{self.name}-operational",
"dataFormat": "json"
}
]
}
}

class DataMeshPlatform:
"""Data mesh implementation platform"""

def __init__(self):
self.data_products: Dict[str, DataProduct] = {}
self.domains: Dict[str, List[str]] = {}

def register_data_product(self, product: DataProduct):
"""Register a new data product"""
self.data_products[product.name] = product

# Add to domain
if product.domain not in self.domains:
self.domains[product.domain] = []
self.domains[product.domain].append(product.name)

# Create infrastructure for data product
self._create_data_product_infra(product)

print(f"Registered data product: {product.name} in domain: {product.domain}")

def _create_data_product_infra(self, product: DataProduct):
"""Create infrastructure for data product"""
# Create S3 bucket for data product
import boto3
s3 = boto3.client('s3')

bucket_name = f"dataproduct-{product.domain}-{product.name}"
s3.create_bucket(Bucket=bucket_name)

# Create Glue database and tables
glue = boto3.client('glue')
database_name = f"dataproduct_{product.domain}_{product.name}"

try:
glue.create_database(DatabaseInput={'Name': database_name})
except glue.exceptions.AlreadyExistsException:
pass

# Create table based on schema
table_input = {
'Name': product.name,
'DatabaseName': database_name,
'TableType': 'EXTERNAL_TABLE',
'Parameters': {
'classification': 'parquet',
'typeOfData': 'file'
},
'StorageDescriptor': {
'Columns': [
{'Name': col_name, 'Type': col_type}
for col_name, col_type in product.schema.items()
],
'Location': f"s3://{bucket_name}/",
'InputFormat': 'org.apache.hadoop.mapred.ParquetInputFormat',
'OutputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat',
'Compressed': False,
'NumberOfBuckets': -1,
'SerdeInfo': {
'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
}
}
}

glue.create_table(DatabaseName=database_name, TableInput=table_input)

# Create API Gateway for data product
self._create_data_product_api(product, bucket_name)

def _create_data_product_api(self, product: DataProduct, bucket_name: str):
"""Create API for data product access"""
import pulumi
import pulumi_aws as aws

# API Gateway
api = aws.apigateway.RestApi(
f"dataproduct-{product.name}-api",
description=f"API for data product: {product.name}"
)

# Lambda function for data access
lambda_func = aws.lambda_.Function(
f"dataproduct-{product.name}-lambda",
runtime="python3.11",
handler="handler.lambda_handler",
code=pulumi.AssetArchive({
".": pulumi.FileArchive("./lambda")
}),
environment={
"variables": {
"BUCKET_NAME": bucket_name,
"DATA_PRODUCT": product.name,
"DOMAIN": product.domain
}
}
)

# API integration
integration = aws.apigateway.LambdaIntegration(
f"dataproduct-{product.name}-integration",
lambda_func.name,
http_method="GET"
)

# Resource and method
resource = api.root.add_resource(product.name)
resource.add_method("GET", integration)

# Deploy
deployment = aws.apigateway.Deployment(
f"dataproduct-{product.name}-deployment",
rest_api=api.id,
stage_name="prod"
)

pulumi.export(f"dataproduct_{product.name}_api_url", deployment.invoke_url)

def discover_data_products(self, domain: Optional[str] = None) -> List[DataProduct]:
"""Discover data products (with optional domain filter)"""
if domain:
return [
self.data_products[name]
for name in self.domains.get(domain, [])
]
else:
return list(self.data_products.values())

def get_data_lineage(self, data_product_name: str) -> Dict:
"""Get lineage for data product"""
# This would integrate with tools like OpenLineage, Marquez
product = self.data_products.get(data_product_name)

if not product:
return {}

lineage = {
"data_product": data_product_name,
"domain": product.domain,
"upstream": self._find_upstream(product),
"downstream": self._find_downstream(product),
"transformations": self._get_transformations(product)
}

return lineage

def enforce_governance(self, data_product_name: str, policy: Dict) -> bool:
"""Enforce governance policies on data product"""
product = self.data_products.get(data_product_name)

if not product:
return False

# Check access policies
for access_policy in product.access_policies:
if not self._check_policy_compliance(access_policy, policy):
return False

# Check quality metrics
for metric_name, threshold in product.quality_metrics.items():
current_value = self._get_current_metric(data_product_name, metric_name)
if current_value < threshold:
print(f"Quality metric {metric_name} below threshold: {current_value} < {threshold}")
return False

return True

# Example usage
platform = DataMeshPlatform()

# Register sales domain data product
sales_product = DataProduct(
name="customer_orders",
domain="sales",
type=DataProductType.SOURCE_ALIGNED,
owner="sales-team@company.com",
sla="99.9% availability, <1 hour freshness",
schema={
"order_id": "string",
"customer_id": "string",
"order_date": "timestamp",
"amount": "decimal(10,2)",
"status": "string"
},
quality_metrics={
"completeness": 0.99,
"freshness_hours": 1,
"validity": 0.95
},
access_policies=[
{
"principal": "sales-analytics-team",
"action": "read",
"resource": "customer_orders"
}
]
)

platform.register_data_product(sales_product)

# Discover all data products in sales domain
sales_products = platform.discover_data_products(domain="sales")
for product in sales_products:
print(f"Found: {product.name}")

6. Cost Optimization & FinOps for Data Platforms​

# Compare: On-prem fixed costs vs Cloud variable costs
# On-prem: Capital expenditure, fixed costs
# Cloud: Operational expenditure, variable costs, need FinOps

import boto3
from datetime import datetime, timedelta
import pandas as pd
import matplotlib.pyplot as plt
from typing import Dict, List
import json

class DataPlatformFinOps:
"""FinOps for cloud data platforms"""

def __init__(self):
self.ce = boto3.client('ce') # Cost Explorer
self.cloudwatch = boto3.client('cloudwatch')

def analyze_data_platform_costs(self, start_date: str, end_date: str) -> Dict:
"""Analyze costs of data platform services"""

# Get cost and usage data
response = self.ce.get_cost_and_usage(
TimePeriod={'Start': start_date, 'End': end_date},
Granularity='DAILY',
Metrics=['UnblendedCost', 'UsageQuantity'],
GroupBy=[
{'Type': 'DIMENSION', 'Key': 'SERVICE'},
{'Type': 'DIMENSION', 'Key': 'USAGE_TYPE'}
]
)

# Parse results
costs_by_service = {}
for result in response['ResultsByTime']:
for group in result['Groups']:
service = group['Keys'][0]
usage_type = group['Keys'][1]
cost = float(group['Metrics']['UnblendedCost']['Amount'])

if service not in costs_by_service:
costs_by_service[service] = {}

if usage_type not in costs_by_service[service]:
costs_by_service[service][usage_type] = 0

costs_by_service[service][usage_type] += cost

# Filter for data platform services
data_services = {
'Amazon S3': costs_by_service.get('Amazon Simple Storage Service', {}),
'Amazon EMR': costs_by_service.get('Amazon Elastic MapReduce', {}),
'Amazon Athena': costs_by_service.get('Amazon Athena', {}),
'Amazon Redshift': costs_by_service.get('Amazon Redshift', {}),
'AWS Glue': costs_by_service.get('AWS Glue', {}),
'Amazon MSK': costs_by_service.get('Amazon Managed Streaming for Apache Kafka', {})
}

return data_services

def optimize_s3_costs(self, bucket_name: str) -> Dict:
"""Optimize S3 storage costs"""
s3 = boto3.client('s3')

# Analyze storage classes
response = s3.list_objects_v2(Bucket=bucket_name)

file_analysis = []
for obj in response.get('Contents', []):
last_accessed = obj.get('LastAccessed', datetime.now())
days_since_access = (datetime.now() - last_accessed.replace(tzinfo=None)).days

file_analysis.append({
'key': obj['Key'],
'size_gb': obj['Size'] / (1024**3),
'last_accessed': last_accessed,
'days_since_access': days_since_access,
'recommended_storage_class': self._recommend_storage_class(days_since_access)
})

# Create recommendations
recommendations = {
'immediate': [],
'near_future': [],
'analysis': file_analysis
}

for file in file_analysis:
if file['recommended_storage_class'] == 'GLACIER':
recommendations['immediate'].append({
'action': 'transition_to_glacier',
'key': file['key'],
'estimated_savings': file['size_gb'] * 0.023 # $0.023/GB/month savings
})
elif file['recommended_storage_class'] == 'STANDARD_IA':
recommendations['near_future'].append({
'action': 'transition_to_standard_ia',
'key': file['key'],
'estimated_savings': file['size_gb'] * 0.012 # $0.012/GB/month savings
})

return recommendations

def _recommend_storage_class(self, days_since_access: int) -> str:
"""Recommend S3 storage class based on access patterns"""
if days_since_access > 180:
return 'GLACIER'
elif days_since_access > 30:
return 'STANDARD_IA'
else:
return 'STANDARD'

def right_size_clusters(self, cluster_type: str, metrics: Dict) -> Dict:
"""Right-size compute clusters based on usage"""

recommendations = []

if cluster_type == 'EMR':
# Analyze CPU and memory usage
avg_cpu_utilization = metrics.get('avg_cpu_utilization', 0)
avg_memory_utilization = metrics.get('avg_memory_utilization', 0)

current_instance_type = metrics.get('instance_type', 'm5.xlarge')
current_count = metrics.get('instance_count', 3)

# Recommendation logic
if avg_cpu_utilization < 30:
# Downsize instance type
new_instance_type = self._get_smaller_instance(current_instance_type)
savings = self._calculate_savings(current_instance_type, new_instance_type, current_count)

recommendations.append({
'action': 'downsize_instance_type',
'current': current_instance_type,
'recommended': new_instance_type,
'estimated_savings_monthly': savings,
'reason': f'Low CPU utilization: {avg_cpu_utilization}%'
})

if avg_memory_utilization < 40:
# Reduce instance count
new_count = max(1, current_count - 1)
savings = self._calculate_savings(current_instance_type, current_instance_type,
current_count - new_count)

recommendations.append({
'action': 'reduce_instance_count',
'current': current_count,
'recommended': new_count,
'estimated_savings_monthly': savings,
'reason': f'Low memory utilization: {avg_memory_utilization}%'
})

elif cluster_type == 'Redshift':
# Analyze query patterns and storage
pass

return recommendations

def implement_cost_governance(self, budget: float) -> Dict:
"""Implement cost governance with alerts and quotas"""

# Create budget
budgets = boto3.client('budgets')

budget_response = budgets.create_budget(
AccountId=boto3.client('sts').get_caller_identity()['Account'],
Budget={
'BudgetName': 'DataPlatformMonthlyBudget',
'BudgetLimit': {
'Amount': str(budget),
'Unit': 'USD'
},
'CostTypes': {
'IncludeTax': False,
'IncludeSubscription': True,
'UseBlended': False
},
'TimeUnit': 'MONTHLY',
'BudgetType': 'COST'
},
NotificationsWithSubscribers=[
{
'Notification': {
'NotificationType': 'ACTUAL',
'ComparisonOperator': 'GREATER_THAN',
'Threshold': 80, # 80% of budget
'ThresholdType': 'PERCENTAGE'
},
'Subscribers': [
{
'SubscriptionType': 'EMAIL',
'Address': 'data-platform-alerts@company.com'
}
]
}
]
)

# Create cost allocation tags
tags_response = self.ce.create_cost_allocation_tags(
CostAllocationTags=[
{
'TagKey': 'DataProduct',
'Type': 'UserDefined'
},
{
'TagKey': 'Environment',
'Type': 'UserDefined'
},
{
'TagKey': 'Team',
'Type': 'UserDefined'
}
]
)

return {
'budget_created': budget_response['ResponseMetadata']['HTTPStatusCode'] == 200,
'tags_created': tags_response['ResponseMetadata']['HTTPStatusCode'] == 200,
'budget_id': budget_response.get('BudgetName', 'Unknown')
}

def generate_cost_dashboard(self) -> str:
"""Generate cost optimization dashboard"""

# Get cost data
end_date = datetime.now().strftime('%Y-%m-%d')
start_date = (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')

costs = self.analyze_data_platform_costs(start_date, end_date)

# Create visualization
fig, axes = plt.subplots(2, 2, figsize=(15, 10))

# 1. Cost by service
services = list(costs.keys())
total_costs = [sum(costs[service].values()) for service in services]

axes[0, 0].bar(services, total_costs)
axes[0, 0].set_title('Monthly Cost by Service')
axes[0, 0].set_ylabel('Cost ($)')
axes[0, 0].tick_params(axis='x', rotation=45)

# 2. Cost trends
# ... (would get daily costs)

# 3. Optimization opportunities
opportunities = {
'S3 Storage Optimization': 1200,
'EMR Right-sizing': 800,
'Redshift Pause/Resume': 1500,
'Glue Job Optimization': 600
}

axes[1, 0].bar(opportunities.keys(), opportunities.values())
axes[1, 0].set_title('Monthly Optimization Opportunities')
axes[1, 0].set_ylabel('Potential Savings ($)')
axes[1, 0].tick_params(axis='x', rotation=45)

plt.tight_layout()

# Save to S3
s3 = boto3.client('s3')
bucket_name = 'cost-dashboards'

import io
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)

key = f"dashboard-{datetime.now().strftime('%Y-%m-%d')}.png"
s3.put_object(Bucket=bucket_name, Key=key, Body=buf.getvalue())

plt.close()

return f"s3://{bucket_name}/{key}"

🎯 Why This Chapter is the FINAL PIECE:​

The Complete Data/AI Engineer Evolution:​

# Your Journey:
stages = [
"Java Developer (Traditional)", # ✓ You started here
"Python Data Engineer (PySpark)", # ✓ Part 5.1
"Pipeline Orchestrator (Airflow)", # ✓ Part 5.2
"MLOps Engineer (Model Serving)", # ✓ Part 5.3
"Cloud Data/AI Architect (THIS CHAPTER)", # ← Final stage
]

# Salary Progression (US):
salary_progression = {
"Java Developer": 120000,
"Data Engineer": 140000,
"Senior Data Engineer": 160000,
"MLOps Engineer": 180000,
"Cloud Data Architect": 220000, # +83% from starting!
}

# Job Market Reality:
"""
The market has plenty of:
- Java Developers
- Data Engineers
- Even MLOps Engineers now

But SEVERE shortage of:
- Cloud-Native Data/AI Architects
- People who understand BOTH data AND infrastructure
- Engineers who can design ENTIRE platforms
"""

# Your Unique Value Proposition:
"""
1. Java production experience (scaling, reliability, monitoring)
2. Python data/ML skills (PySpark, MLflow, FastAPI)
3. Cloud infrastructure expertise (Kubernetes, Terraform, Cloud Services)
4. Ability to build COMPLETE platforms end-to-end

→ You become UNIQUE in the market
→ You command TOP salaries
→ You get ARCHITECT roles (not just engineer)
"""

Complete Skill Stack After This Chapter:​

✅ Programming: Java (Spring Boot), Python (FastAPI, PySpark)
✅ Data Engineering: Airflow, DBT, Great Expectations
✅ ML Engineering: Scikit-learn, MLflow, Feature Stores
✅ MLOps: Model Serving, Monitoring, Registry
✅ Cloud Infrastructure: AWS/GCP/Azure, Terraform, Pulumi
✅ Kubernetes: Spark on K8s, ML on K8s, Service Mesh
✅ Data Architecture: Data Mesh, Cost Optimization, Governance
✅ Soft Skills: Cross-team collaboration, Architecture design

→ You're now a CLOUD DATA/AI PLATFORM ARCHITECT

📚 Capstone Project: Build a Cloud-Native Data/AI Platform​

# Final Project: Enterprise Data & AI Platform
# Architecture that uses EVERYTHING you've learned:

"""
┌─────────────────────────────────────────────────────────────────┐
│ CLOUD-NATIVE DATA & AI PLATFORM │
├─────────────────────────────────────────────────────────────────┤
│ │
│ Infrastructure Layer (Terraform/Pulumi) │
│ ├─ AWS/GCP/Azure cloud resources │
│ ├─ Kubernetes clusters (EKS/GKE/AKS) │
│ └─ Networking, security, monitoring │
│ │
│ Platform Services Layer (Kubernetes Operators) │
│ ├─ Spark Operator for batch processing │
│ ├─ Airflow for workflow orchestration │
│ ├─ MLflow for experiment tracking & registry │
│ ├─ Feast for feature store │
│ └─ Grafana/Prometheus for monitoring │
│ │
│ Data Products Layer (Data Mesh) │
│ ├─ Domain-specific data products (sales, marketing, etc.) │
│ ├─ Each with own S3 bucket, Glue catalog, API │
│ ├─ Federated governance & quality │
│ └─ Self-service data discovery │
│ │
│ AI/ML Layer (MLOps) │
│ ├─ Feature engineering pipelines │
│ ├─ Model training & evaluation │
│ ├─ Model serving APIs (FastAPI/Triton) │
│ ├─ A/B testing & canary deployments │
│ └─ Model monitoring & drift detection │
│ │
│ Application Layer │
│ ├─ Java microservices (Spring Boot) │
│ ├─ Python services (FastAPI) │
│ ├─ Real-time streaming (Kafka/Flink) │
│ └─ Web applications (React/Vue) │
│ │
│ Cost & Governance Layer (FinOps) │
│ ├─ Cost monitoring & optimization │
│ ├─ Budget alerts & quotas │
│ ├─ Resource tagging & allocation │
│ └─ Compliance & security policies │
└─────────────────────────────────────────────────────────────────┘
"""

# Your Role: Design and implement the ENTIRE platform
# From infrastructure to applications, including cost governance

🚀 Learning Path for Part 5.4:​

  1. Week 1-2: Cloud Data Platforms (AWS/GCP/Azure comparison)
  2. Week 3-4: Infrastructure as Code (Terraform, Pulumi, CDK)
  3. Week 5-6: Kubernetes-native Data Processing (Spark, Dask, Ray on K8s)
  4. Week 7-8: Serverless Data Architectures
  5. Week 9-10: Data Mesh & Federated Architecture
  6. Week 11-12: Cost Optimization & FinOps
  7. Week 13-14: Complete Platform Implementation
  8. Week 15-16: Production Deployment & Optimization

💡 The Ultimate Career Achievement:​

After this chapter, you'll be qualified for:

Senior Cloud Data Engineer ($180k+)
Data Platform Architect ($200k+)
AI Platform Architect ($220k+)
Head of Data Engineering ($250k+)
CTO/VP Engineering ($300k+)

You'll have the COMPLETE skill set to:

  1. Design enterprise data/AI platforms
  2. Implement them from scratch
  3. Optimize for cost and performance
  4. Govern for security and compliance
  5. Lead teams to build and maintain them