CloudWatch

# Create log group
aws logs create-log-group --log-group-name /my-app/production

# Set retention (days: 1,3,5,7,14,30,60,90,120,150,180,365,400,545,731,1827,3653)
aws logs put-retention-policy \
  --log-group-name /my-app/production \
  --retention-in-days 30

# List log groups
aws logs describe-log-groups \
  --query 'logGroups[*].[logGroupName,retentionInDays,storedBytes]' \
  --output table

# List log streams in a group
aws logs describe-log-streams \
  --log-group-name /aws/lambda/my-func \
  --order-by LastEventTime --descending

# Delete log group
aws logs delete-log-group --log-group-name /my-app/production

# Tail in real time (CLI v2)
aws logs tail /aws/lambda/my-func --follow
aws logs tail /aws/lambda/my-func --since 1h
aws logs tail /aws/lambda/my-func --since 2024-01-15T10:00:00

# Filter log events (older/simpler)
aws logs filter-log-events \
  --log-group-name /aws/lambda/my-func \
  --filter-pattern "ERROR" \
  --start-time $(($(date +%s) - 3600))000

# CloudWatch Logs Insights query
aws logs start-query \
  --log-group-name /aws/lambda/my-func \
  --start-time $(($(date +%s) - 3600)) \
  --end-time $(date +%s) \
  --query-string 'fields @timestamp, @message | filter @message like /ERROR/ | sort @timestamp desc | limit 20'

# Get query results (use queryId from above)
aws logs get-query-results --query-id abc-123-def

# Top 10 error messages
fields @message
| filter @message like /ERROR/
| stats count(*) as errors by @message
| sort errors desc
| limit 10

# P99 Lambda duration
filter @type = "REPORT"
| stats avg(Duration), pct(Duration, 99) by bin(5m)

# Count requests by status code (API Gateway)
fields status
| stats count(*) as requests by status
| sort requests desc

# Memory usage trend (Lambda)
filter @type = "REPORT"
| fields @memorySize / 1024 / 1024 as memMB, MaxMemoryUsed / 1024 / 1024 as usedMB
| stats avg(usedMB), max(usedMB) by bin(1h)

# Put custom metric
aws cloudwatch put-metric-data \
  --namespace MyApp \
  --metric-name OrderCount \
  --value 5 \
  --unit Count \
  --dimensions Service=checkout,Env=prod

# Get metric statistics
aws cloudwatch get-metric-statistics \
  --namespace AWS/Lambda \
  --metric-name Duration \
  --dimensions Name=FunctionName,Value=my-func \
  --start-time 2024-01-15T00:00:00Z \
  --end-time   2024-01-15T01:00:00Z \
  --period 300 \
  --statistics Average Maximum

# List available metrics
aws cloudwatch list-metrics --namespace AWS/EC2
aws cloudwatch list-metrics --namespace MyApp

# Alarm on Lambda error rate
aws cloudwatch put-metric-alarm \
  --alarm-name lambda-errors-high \
  --alarm-description "Lambda errors > 5 in 5 min" \
  --namespace AWS/Lambda \
  --metric-name Errors \
  --dimensions Name=FunctionName,Value=my-func \
  --statistic Sum \
  --period 300 \
  --threshold 5 \
  --comparison-operator GreaterThanOrEqualToThreshold \
  --evaluation-periods 1 \
  --alarm-actions arn:aws:sns:us-east-1:123:my-topic \
  --ok-actions arn:aws:sns:us-east-1:123:my-topic \
  --treat-missing-data notBreaching

# Alarm on EC2 CPU > 80%
aws cloudwatch put-metric-alarm \
  --alarm-name ec2-cpu-high \
  --namespace AWS/EC2 \
  --metric-name CPUUtilization \
  --dimensions Name=InstanceId,Value=i-0abc123 \
  --statistic Average \
  --period 300 \
  --threshold 80 \
  --comparison-operator GreaterThanOrEqualToThreshold \
  --evaluation-periods 2 \
  --alarm-actions arn:aws:sns:us-east-1:123:ops-topic

# Composite alarm (combines multiple alarms with AND/OR)
aws cloudwatch put-composite-alarm \
  --alarm-name service-degraded \
  --alarm-rule "ALARM(lambda-errors-high) OR ALARM(ec2-cpu-high)" \
  --alarm-actions arn:aws:sns:us-east-1:123:pagerduty-topic

aws cloudwatch describe-alarms --alarm-names lambda-errors-high
aws cloudwatch set-alarm-state \
  --alarm-name lambda-errors-high \
  --state-value ALARM \
  --state-reason "Testing"
aws cloudwatch delete-alarms --alarm-names lambda-errors-high

# Create dashboard from JSON body
aws cloudwatch put-dashboard \
  --dashboard-name MyServiceDashboard \
  --dashboard-body file://dashboard.json

aws cloudwatch list-dashboards
aws cloudwatch get-dashboard --dashboard-name MyServiceDashboard
aws cloudwatch delete-dashboards --dashboard-names MyServiceDashboard

# Install on Amazon Linux 2
sudo yum install -y amazon-cloudwatch-agent

# Wizard to generate config
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-config-wizard

# Start with config file
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a fetch-config -m ec2 \
  -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \
  -s

# Status
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a status

{
  "metrics": {
    "namespace": "MyEC2Metrics",
    "metrics_collected": {
      "mem":  { "measurement": ["mem_used_percent"] },
      "disk": {
        "measurement": ["used_percent"],
        "resources": ["/"]
      }
    }
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [{
          "file_path":       "/var/log/my-app/*.log",
          "log_group_name":  "/my-app/production",
          "log_stream_name": "{instance_id}",
          "timestamp_format": "%Y-%m-%dT%H:%M:%S"
        }]
      }
    }
  }
}

# Schedule Lambda every 5 minutes (EventBridge rule)
aws events put-rule \
  --name run-every-5min \
  --schedule-expression "rate(5 minutes)" \
  --state ENABLED

# Add Lambda as target
aws events put-targets \
  --rule run-every-5min \
  --targets "Id=1,Arn=arn:aws:lambda:us-east-1:123:function:my-func"

# Grant EventBridge permission to invoke Lambda
aws lambda add-permission \
  --function-name my-func \
  --statement-id eventbridge-invoke \
  --action lambda:InvokeFunction \
  --principal events.amazonaws.com \
  --source-arn arn:aws:events:us-east-1:123:rule/run-every-5min

aws events list-rules
aws events delete-rule --name run-every-5min

# Anomaly detection band
aws cloudwatch put-anomaly-detector \
  --namespace AWS/Lambda \
  --metric-name Duration \
  --dimensions '[{"Name":"FunctionName","Value":"my-func"}]' \
  --stat Average

# Alarm on anomaly
aws cloudwatch put-metric-alarm \
  --alarm-name lambda-duration-anomaly \
  --metrics '[
    {"Id":"m1","MetricStat":{"Metric":{"Namespace":"AWS/Lambda","MetricName":"Duration","Dimensions":[{"Name":"FunctionName","Value":"my-func"}]},"Period":300,"Stat":"Average"}},
    {"Id":"ad1","Expression":"ANOMALY_DETECTION_BAND(m1, 2)","Label":"Expected"}
  ]' \
  --comparison-operator GreaterThanUpperThreshold \
  --threshold-metric-id ad1 \
  --evaluation-periods 2

# Count ERROR occurrences as a custom metric
aws logs put-metric-filter \
  --log-group-name /my-app/production \
  --filter-name error-count \
  --filter-pattern "ERROR" \
  --metric-transformations \
    metricName=ErrorCount,metricNamespace=MyApp,metricValue=1,defaultValue=0

aws logs describe-metric-filters --log-group-name /my-app/production