# Check current resource usage trước khi quyết định
kubectl top pods -n production
kubectl top nodes
 
# Database connection usage
aws cloudwatch get-metric-statistics \
  --namespace AWS/RDS \
  --metric-name DatabaseConnections \
  --dimensions Name=DBInstanceIdentifier,Value=prod-db \
  --statistics Maximum Average \
  --period 3600 \
  --start-time 2026-04-07T00:00:00Z \
  --end-time 2026-04-08T00:00:00Z

                    ┌──────────────────────┐
                    │    ALB (AWS)          │
                    │  Round-Robin / Least  │
                    │    Connections        │
                    └──────────┬───────────┘
                               │
              ┌────────────────┼────────────────┐
              │                │                │
    ┌─────────▼──────┐ ┌──────▼────────┐ ┌─────▼─────────┐
    │  App Pod 1     │ │  App Pod 2    │ │  App Pod 3    │
    │  (stateless)   │ │  (stateless)  │ │  (stateless)  │
    └─────────┬──────┘ └──────┬────────┘ └─────┬─────────┘
              │                │                │
              └────────────────┼────────────────┘
                               │
                    ┌──────────▼───────────┐
                    │   Redis (Session +   │
                    │   Cache)             │
                    └──────────────────────┘

// Spring Boot — application.yml
spring:
  session:
    store-type: redis
    timeout: 30m
  data:
    redis:
      host: prod-redis.xxxxx.cache.amazonaws.com
      port: 6379
 
// Trước: session lưu local — user bị logout khi request đến server khác
// Sau: bất kỳ server nào cũng đọc được session từ Redis

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-deployment
  minReplicas: 3
  maxReplicas: 10
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 60
    - type: Resource
      resource:
        name: memory
        target:
          type: Utilization
          averageUtilization: 70

              App Servers
              /         \
      Write queries    Read queries
            |              |
    ┌───────▼──────┐  ┌───▼───────────┐
    │ RDS Primary  │  │ RDS Read      │
    │ (Write)      │──│ Replica       │
    │ db.r6g.xl    │  │ db.r6g.xl     │
    └──────────────┘  └───────────────┘

@Configuration
public class DataSourceConfig {
 
    @Bean
    @Primary
    public DataSource routingDataSource(
            @Qualifier("primaryDS") DataSource primary,
            @Qualifier("replicaDS") DataSource replica) {
 
        var routing = new AbstractRoutingDataSource() {
            @Override
            protected Object determineCurrentLookupKey() {
                return TransactionSynchronizationManager
                    .isCurrentTransactionReadOnly() ? "replica" : "primary";
            }
        };
        routing.setTargetDataSources(Map.of(
            "primary", primary,
            "replica", replica
        ));
        routing.setDefaultTargetDataSource(primary);
        return routing;
    }
}
 
// Service layer — đánh dấu read-only
@Service
public class ProductService {
 
    @Transactional(readOnly = true)  // → route sang replica
    public List<Product> findPopular(int limit) {
        return productRepository.findTopByOrderBySoldDesc(limit);
    }
 
    @Transactional  // → route sang primary
    public Product create(CreateProductRequest req) {
        return productRepository.save(Product.from(req));
    }
}

-- Tìm slow queries
SELECT query, calls, mean_exec_time, total_exec_time
FROM pg_stat_statements
ORDER BY mean_exec_time DESC
LIMIT 20;
 
-- Thêm index cho query phổ biến
CREATE INDEX CONCURRENTLY idx_orders_user_created
ON orders(user_id, created_at DESC);
 
-- Kiểm tra index có được dùng không
EXPLAIN ANALYZE
SELECT * FROM orders WHERE user_id = 123 ORDER BY created_at DESC LIMIT 10;

@Service
public class ProductService {
 
    @Cacheable(value = "products", key = "#id", unless = "#result == null")
    public Product findById(Long id) {
        return productRepository.findById(id).orElse(null);
    }
 
    @CacheEvict(value = "products", key = "#product.id")
    public Product update(Product product) {
        return productRepository.save(product);
    }
 
    @Cacheable(value = "popular-products", key = "'top-' + #limit")
    @Transactional(readOnly = true)
    public List<Product> findPopular(int limit) {
        return productRepository.findTopByOrderBySoldDesc(limit);
    }
}

# application.yml
spring:
  cache:
    type: redis
    redis:
      time-to-live: 60s      # TTL mặc định
      cache-null-values: false

             User Request
                  │
    ┌─────────────▼──────────────┐
    │  App Server                │
    │  1. Validate & save order  │──── Response ngay (200ms)
    │  2. Publish event to Kafka │
    └─────────────┬──────────────┘
                  │ (async)
    ┌─────────────▼──────────────┐
    │  Kafka Topic: order-events │
    └─────────────┬──────────────┘
                  │
      ┌───────────┼────────────┐
      │           │            │
  ┌───▼───┐  ┌───▼───┐  ┌─────▼────┐
  │ Email │  │ Inven │  │ Analytics│
  │ Worker│  │ Worker│  │ Worker   │
  └───────┘  └───────┘  └──────────┘

// Producer — publish event sau khi tạo order
@Service
@RequiredArgsConstructor
public class OrderService {
 
    private final KafkaTemplate<String, OrderEvent> kafkaTemplate;
 
    @Transactional
    public Order createOrder(CreateOrderRequest req) {
        Order order = orderRepository.save(Order.from(req));
 
        // Async — user không chờ email, inventory update, analytics
        kafkaTemplate.send("order-events",
            order.getId().toString(),
            new OrderEvent(order.getId(), OrderEventType.CREATED));
 
        return order;  // Response ngay cho user
    }
}
 
// Consumer — xử lý background
@Component
public class OrderEventConsumer {
 
    @KafkaListener(topics = "order-events", groupId = "email-service")
    public void handleOrderEvent(OrderEvent event) {
        if (event.type() == OrderEventType.CREATED) {
            emailService.sendOrderConfirmation(event.orderId());
        }
    }
}

# Prometheus alerts cho hệ thống 50K users
groups:
  - name: scaling-alerts
    rules:
      # App tier
      - alert: HighCPUUsage
        expr: |
          avg(rate(container_cpu_usage_seconds_total{namespace="production"}[5m]))
            by (pod) > 0.8
        for: 10m
        annotations:
          summary: "Pod {{ $labels.pod }} CPU > 80% — cân nhắc scale out"
 
      # Database
      - alert: HighDBConnections
        expr: |
          pg_stat_activity_count{datname="production"}
            / pg_settings_max_connections > 0.7
        for: 5m
        annotations:
          summary: "DB connections > 70% max — cân nhắc connection pooling hoặc read replica"
 
      # Slow queries
      - alert: SlowQueries
        expr: |
          rate(pg_stat_statements_mean_exec_time_seconds{datname="production"}[5m]) > 0.5
        for: 5m
        annotations:
          summary: "Avg query time > 500ms — check slow query log"
 
      # Cache
      - alert: LowCacheHitRate
        expr: |
          redis_keyspace_hits_total
            / (redis_keyspace_hits_total + redis_keyspace_misses_total) < 0.85
        for: 15m
        annotations:
          summary: "Cache hit rate < 85% — review caching strategy"
 
      # Kafka consumer lag
      - alert: HighConsumerLag
        expr: kafka_consumergroup_lag_sum > 10000
        for: 10m
        annotations:
          summary: "Kafka consumer lag > 10K — cần thêm consumer instances"