Uploaded image for project: 'ONAP Operations Manager'
  1. ONAP Operations Manager
  2. OOM-542

two pods not starting under OOM (CrashLoopBackOff) - under Rancher 1.6.12

XMLWordPrintable

      fix: Use Rancher 1.6.10 (avoid 1.6.12 or 1.6.13 until we solve issues related to OOM-431 and OOM-441
      ------------------------------------
      Basic setup, two hosts

      host1 : devstack

      host2 : rancher, kubernetes, oom (onap)

      The following containers fail to start: 

      NAMESPACE NAME READY STATUS RESTARTS AGE
      onap-log elasticsearch-6df4f65775-w48c9 0/1 CrashLoopBackOff 28 2h
      onap-portal vnc-portal-845d84676c-jwzf9 0/1 CrashLoopBackOff 23 2h

      $ kubectl describe -n onap-log po elasticsearch-6df4f65775-w48c9
      Name: elasticsearch-6df4f65775-w48c9
      Namespace: onap-log
      Node: ip-172-31-37-207.us-west-2.compute.internal/35.161.247.60
      Start Time: Wed, 03 Jan 2018 18:44:59 +0000
      Labels: app=elasticsearch
      {{ pod-template-hash=2890921331}}
      Annotations: kubernetes.io/created-by={"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"ReplicaSet","namespace":"onap-log","name":"elasticsearch-6df4f65775","uid":"3336be76-f0b6-11e7-9c76-029aeb...
      Status: Running
      IP: 10.42.66.68
      Created By: ReplicaSet/elasticsearch-6df4f65775
      Controlled By: ReplicaSet/elasticsearch-6df4f65775
      Containers:
      {{ elasticsearch:}}
      {{ Container ID: docker://73252edb5572ce4d1a3267a1e2b8e52c1de980a2ecaf464d142ad54c472c403a}}
      {{ Image: docker.elastic.co/elasticsearch/elasticsearch:5.5.0}}
      {{ Image ID: docker-pullable://docker.elastic.co/elasticsearch/elasticsearch@sha256:80722d5c10da8bd33e9f32cb27cf5a3f23a96fbbb0adeb89e6cb724bbab9d217}}
      {{ Ports: 9200/TCP, 9300/TCP}}
      {{ State: Waiting}}
      {{ Reason: CrashLoopBackOff}}
      {{ Last State: Terminated}}
      {{ Reason: Error}}
      {{ Exit Code: 78}}
      {{ Started: Wed, 03 Jan 2018 21:30:03 +0000}}
      {{ Finished: Wed, 03 Jan 2018 21:30:10 +0000}}
      {{ Ready: False}}
      {{ Restart Count: 30}}
      {{ Readiness: tcp-socket :9300 delay=0s timeout=1s period=10s #success=1 #failure=3}}
      {{ Environment: <none>}}
      {{ Mounts:}}
      {{ /usr/share/elasticsearch/config/elasticsearch.yml from elasticsearch-conf (rw)}}
      {{ /usr/share/elasticsearch/data/ from elasticsearch-data (rw)}}
      {{ /usr/share/elasticsearch/logs/ from elasticsearch-logs (rw)}}
      {{ /var/run/secrets/kubernetes.io/serviceaccount from default-token-cl74v (ro)}}
      Conditions:
      {{ Type Status}}
      {{ Initialized True}}
      {{ Ready False}}
      {{ PodScheduled True}}
      Volumes:
      {{ elasticsearch-data:}}
      {{ Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)}}
      {{ ClaimName: elasticsearch-db}}
      {{ ReadOnly: false}}
      {{ elasticsearch-logs:}}
      {{ Type: HostPath (bare host directory volume)}}
      {{ Path: /dockerdata-nfs/onap/log/elasticsearch/logs}}
      {{ elasticsearch-conf:}}
      {{ Type: ConfigMap (a volume populated by a ConfigMap)}}
      {{ Name: elasticsearch-configmap}}
      {{ Optional: false}}
      {{ default-token-cl74v:}}
      {{ Type: Secret (a volume populated by a Secret)}}
      {{ SecretName: default-token-cl74v}}
      {{ Optional: false}}
      QoS Class: BestEffort
      Node-Selectors: <none>
      Tolerations: node.alpha.kubernetes.io/notReady:NoExecute for 300s
      {{ node.alpha.kubernetes.io/unreachable:NoExecute for 300s}}
      Events:
      {{ Type Reason Age From Message}}
      {{ ---- ------ ---- ---- -------}}
      {{ Warning FailedSync 3m (x576 over 2h) kubelet, ip-172-31-37-207.us-west-2.compute.internal Error syncing pod}}

       

      $ kubectl describe -n onap-portal po vnc-portal-845d84676c-jwzf9
      Name: vnc-portal-845d84676c-jwzf9
      Namespace: onap-portal
      Node: ip-172-31-37-207.us-west-2.compute.internal/35.161.247.60
      Start Time: Wed, 03 Jan 2018 18:44:44 +0000
      Labels: app=vnc-portal
      {{ pod-template-hash=4018402327}}
      Annotations: kubernetes.io/created-by={"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"ReplicaSet","namespace":"onap-portal","name":"vnc-portal-845d84676c","uid":"2a573cc2-f0b6-11e7-9c76-029aeb...
      Status: Running
      IP: 10.42.12.205
      Created By: ReplicaSet/vnc-portal-845d84676c
      Controlled By: ReplicaSet/vnc-portal-845d84676c
      Containers:
      {{ vnc-portal:}}
      {{ Container ID: docker://4c632d2e0ded752b696ea63636344576a379c14077867c0a5d0d1078527d090d}}
      {{ Image: dorowu/ubuntu-desktop-lxde-vnc}}
      {{ Image ID: docker-pullable://dorowu/ubuntu-desktop-lxde-vnc@sha256:9e79e40c997ea6c0789ce695e9530ea13140f561669c1cacd50dbe054289c0cd}}
      {{ Port: <none>}}
      {{ State: Waiting}}
      {{ Reason: CrashLoopBackOff}}
      {{ Last State: Terminated}}
      {{ Reason: Error}}
      {{ Exit Code: 137}}
      {{ Started: Wed, 03 Jan 2018 21:30:33 +0000}}
      {{ Finished: Wed, 03 Jan 2018 21:31:04 +0000}}
      {{ Ready: False}}
      {{ Restart Count: 25}}
      {{ Environment:}}
      {{ VNC_PASSWORD: password}}
      {{ Mounts:}}
      {{ /etc/localtime from localtime (ro)}}
      {{ /root/.init_profile/profiles.ini from vnc-profiles-ini (rw)}}
      {{ /ubuntu-init/ from ubuntu-init (rw)}}
      {{ /var/run/secrets/kubernetes.io/serviceaccount from default-token-mhkvx (ro)}}
      Conditions:
      {{ Type Status}}
      {{ Initialized True}}
      {{ Ready False}}
      {{ PodScheduled True}}
      Volumes:
      {{ localtime:}}
      {{ Type: HostPath (bare host directory volume)}}
      {{ Path: /etc/localtime}}
      {{ ubuntu-init:}}
      {{ Type: EmptyDir (a temporary directory that shares a pod's lifetime)}}
      {{ Medium:}}
      {{ vnc-profiles-ini:}}
      {{ Type: ConfigMap (a volume populated by a ConfigMap)}}
      {{ Name: vnc-profiles-ini}}
      {{ Optional: false}}
      {{ default-token-mhkvx:}}
      {{ Type: Secret (a volume populated by a Secret)}}
      {{ SecretName: default-token-mhkvx}}
      {{ Optional: false}}
      QoS Class: BestEffort
      Node-Selectors: <none>
      Tolerations: node.alpha.kubernetes.io/notReady:NoExecute for 300s
      {{ node.alpha.kubernetes.io/unreachable:NoExecute for 300s}}
      Events:
      {{ Type Reason Age From Message}}
      {{ ---- ------ ---- ---- -------}}
      {{ Warning BackOff 55m (x228 over 1h) kubelet, ip-172-31-37-207.us-west-2.compute.internal Back-off restarting failed container}}
      {{ Normal Pulled 25m (x22 over 2h) kubelet, ip-172-31-37-207.us-west-2.compute.internal Successfully pulled image "dorowu/ubuntu-desktop-lxde-vnc"}}
      {{ Normal Pulling 20m (x23 over 2h) kubelet, ip-172-31-37-207.us-west-2.compute.internal pulling image "dorowu/ubuntu-desktop-lxde-vnc"}}
      {{ Warning FailedSync 5m (x476 over 2h) kubelet, ip-172-31-37-207.us-west-2.compute.internal Error syncing pod}}

            michaelobrien michaelobrien
            dbainbri dbainbri
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

              Created:
              Updated:
              Resolved:

                Estimated:
                Original Estimate - Not Specified
                Not Specified
                Remaining:
                Remaining Estimate - 0 minutes
                0m
                Logged:
                Time Spent - 2 hours
                2h