A&AI healthcheck timed out after OOM deployment. Some pods failed on deployment:

root@release-rancher:~# kubectl -n onap get pod |grep aai
dev-aai-aai-7ddd6b848f-rdd9w 0/1 Init:0/1 54 9h
dev-aai-aai-babel-7cb986869d-k5nb2 2/2 Running 0 9h
dev-aai-aai-cassandra-0 1/1 Running 0 9h
dev-aai-aai-cassandra-1 1/1 Running 1 9h
dev-aai-aai-cassandra-2 1/1 Running 0 9h
dev-aai-aai-champ-6b6d4779d8-kvxnv 2/2 Running 0 9h
dev-aai-aai-data-router-6c678645d-7h6rt 2/2 Running 0 9h
dev-aai-aai-elasticsearch-669f4664c-2jc7m 1/1 Running 0 9h
dev-aai-aai-gizmo-9df94f898-99rmn 2/2 Running 0 9h
dev-aai-aai-graphadmin-9b874c4f7-gv8sn 0/2 Init:0/1 55 9h
dev-aai-aai-graphadmin-create-db-schema-4kxz2 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-4tvsf 0/1 Error 0 9h
dev-aai-aai-graphadmin-create-db-schema-6mjgc 0/1 Error 0 9h
dev-aai-aai-graphadmin-create-db-schema-79n6j 0/1 Error 0 7h
dev-aai-aai-graphadmin-create-db-schema-82gb5 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-b9g4g 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-bw865 0/1 Error 0 9h
dev-aai-aai-graphadmin-create-db-schema-bxx9z 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-cvn5z 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-czwb6 0/1 Error 0 9h
dev-aai-aai-graphadmin-create-db-schema-k6mxw 0/1 Error 0 9h
dev-aai-aai-graphadmin-create-db-schema-kg4nz 0/1 Error 0 9h
dev-aai-aai-graphadmin-create-db-schema-lwmvb 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-pfrx5 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-snj98 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-t2vzk 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-t8h9w 0/1 Error 0 8h
dev-aai-aai-graphadmin-create-db-schema-v67qg 0/1 Error 0 9h
dev-aai-aai-modelloader-54f7444dcd-snv7c 2/2 Running 0 9h
dev-aai-aai-resources-8579748958-xv7sq 0/2 Init:0/1 55 9h
dev-aai-aai-search-data-695b878655-spxfx 2/2 Running 0 9h
dev-aai-aai-sparky-be-bb797db78-9dcgw 0/2 Init:0/1 0 9h
dev-aai-aai-spike-59c6db6574-gwk4z 2/2 Running 0 9h
dev-aai-aai-traversal-7f5669d56-cls6j 0/2 Init:0/1 55 9h
dev-aai-aai-traversal-update-query-data-bd56m 0/1 Init:0/1 55 9h

root@release-rancher:~# kubectl -n onap describe pod dev-aai-aai-graphadmin-create-db-schema-v67qg
Name: dev-aai-aai-graphadmin-create-db-schema-v67qg
Namespace: onap
Node: release-k8s-01/10.0.0.21
Start Time: Mon, 14 Jan 2019 04:42:59 +0000
Labels: app=aai-graphadmin-job
controller-uid=0316371c-17b4-11e9-a0fe-024564a340f1
job-name=dev-aai-aai-graphadmin-create-db-schema
release=dev-aai
Annotations: checksum/config=e3ee60e836229418a3b64abeae209b4a236647f09f06440be95e91cb6d17e3c4
Status: Failed
IP: 10.42.221.69
Controlled By: Job/dev-aai-aai-graphadmin-create-db-schema
Init Containers:
aai-graphadmin-readiness:
Container ID: docker://1f27c57801688324252a0cabd0e7d5383fbd57e3d8abb66212cd7f63765a2015
Image: oomk8s/readiness-check:2.0.0
Image ID: docker-pullable://oomk8s/readiness-check@sha256:7daa08b81954360a1111d03364febcb3dcfeb723bcc12ce3eb3ed3e53f2323ed
Port: <none>
Host Port: <none>
Command:
/root/ready.py
Args:
--container-name
aai-cassandra
State: Terminated
Reason: Completed
Exit Code: 0
Started: Mon, 14 Jan 2019 04:43:02 +0000
Finished: Mon, 14 Jan 2019 04:43:05 +0000
Ready: True
Restart Count: 0
Environment:
NAMESPACE: onap (v1:metadata.namespace)
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from default-token-9hnc7 (ro)
Containers:
aai-graphadmin-job:
Container ID: docker://16ce03c0e283e02ec904300484713246e2d9f5609b8ada071762db521988bb42
Image: 10.12.5.2:5000/onap/aai-graphadmin:1.0.3
Image ID: docker-pullable://10.12.5.2:5000/onap/aai-graphadmin@sha256:25c90495ddf2ad0dbc458348995b13c8668b3ceeb540be12cf1ca86b5e3e1301
Port: <none>
Host Port: <none>
Command:
/bin/bash
docker-entrypoint.sh
createDBSchema.sh
State: Terminated
Reason: Error
Exit Code: 1
Started: Mon, 14 Jan 2019 04:43:21 +0000
Finished: Mon, 14 Jan 2019 04:43:57 +0000
Ready: False
Restart Count: 0
Environment:
LOCAL_USER_ID: 1000
LOCAL_GROUP_ID: 1000
Mounts:
/etc/localtime from localtime (ro)
/opt/aai/logroot/AAI-RES from dev-aai-aai-graphadmin-logs (rw)
/opt/app/aai-graphadmin/resources/application.properties from dev-aai-aai-graphadmin-springapp-conf (rw)
/opt/app/aai-graphadmin/resources/etc/appprops/aaiconfig.properties from dev-aai-aai-graphadmin-aaiconfig-conf (rw)
/opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-cached.properties from dev-aai-aai-graphadmin-db-cached-conf (rw)
/opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-realtime.properties from dev-aai-aai-graphadmin-db-real-conf (rw)
/opt/app/aai-graphadmin/resources/etc/auth/aai_keystore from dev-aai-aai-graphadmin-auth-truststore-sec (rw)
/opt/app/aai-graphadmin/resources/localhost-access-logback.xml from dev-aai-aai-graphadmin-localhost-access-log-conf (rw)
/opt/app/aai-graphadmin/resources/logback.xml from dev-aai-aai-graphadmin-log-conf (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-9hnc7 (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
localtime:
Type: HostPath (bare host directory volume)
Path: /etc/localtime
HostPathType:
filebeat-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: aai-filebeat
Optional: false
dev-aai-aai-graphadmin-logs:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
dev-aai-aai-graphadmin-filebeat:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
dev-aai-aai-graphadmin-log-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-aai-aai-graphadmin-log
Optional: false
dev-aai-aai-graphadmin-localhost-access-log-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-aai-aai-graphadmin-localhost-access-log-configmap
Optional: false
dev-aai-aai-graphadmin-db-real-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-aai-aai-graphadmin-db-real-configmap
Optional: false
dev-aai-aai-graphadmin-db-cached-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-aai-aai-graphadmin-db-cached-configmap
Optional: false
dev-aai-aai-graphadmin-aaiconfig-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-aai-aai-graphadmin-aaiconfig-configmap
Optional: false
dev-aai-aai-graphadmin-springapp-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-aai-aai-graphadmin-springapp-configmap
Optional: false
dev-aai-aai-graphadmin-realm-conf:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-aai-aai-graphadmin-realm-configmap
Optional: false
dev-aai-aai-graphadmin-auth-truststore-sec:
Type: Secret (a volume populated by a Secret)
SecretName: aai-auth-truststore-secret
Optional: false
default-token-9hnc7:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-9hnc7
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events: <none>