20171013 - verify fix in https://gerrit.onap.org/r/#/c/18803/3

 

Triaging cause of HC failure - will debug line 61 to be sure

 

Mike mentioned an SDC chef error - looks like robot healthcheck fails on the FE container which looks ok - but the underlying call to the BE fails with a 500 - looks like there is a chef issue - looking at the actual script now

 

FE logs are ok

root@ip-172-31-82-46:/dockerdata-nfs/onap/sdc/logs# kubectl logs -n onap-sdc sdc-fe-1573125197-lmb2b sdc-fe

BE logs have a chef error

STDERR:

---- End output of "bash"  "/tmp/jetty/chef-script20171008-1280-17vy4b8" ----

Ran "bash"  "/tmp/jetty/chef-script20171008-1280-17vy4b8" returned 1

[2017-10-08T15:11:40+00:00] FATAL: Chef::Exceptions::ChildConvergeError: Chef run process exited unsuccessfully (exit code 1)

root@ip-172-31-82-46:/dockerdata-nfs/onap/sdc/logs# kubectl logs -n onap-sdc sdc-be-2986438255-9fksq sdc-be

 

 

 

2017-10-11 01:10:56.207:WARN:oejs.HttpChannel:qtp142638629-1532528: //sdc-be.onap-sdc:8080/sdc2/rest/healthCheck

javax.servlet.ServletException: javax.servlet.ServletException: java.lang.NullPointerException

at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:139)

at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)

at org.eclipse.jetty.server.Server.handle(Server.java:534)

at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)

at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)

at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)

at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)

at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)

at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)

at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)

at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)

at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)

at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)

at java.lang.Thread.run(Thread.java:748)

Caused by:

javax.servlet.ServletException: java.lang.NullPointerException

at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:489)

at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:427)

at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:388)

at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:341)

at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:228)

at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:840)

at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1772)

at org.eclipse.jetty.websocket.server.WebSocketUpgradeFilter.doFilter(WebSocketUpgradeFilter.java:205)

at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)

at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)

at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)

at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)

at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)

at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)

at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)

at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)

at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)

at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)

at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)

at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)

at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)

at org.eclipse.jetty.server.Server.handle(Server.java:534)

at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)

at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)

at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)

at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)

at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)

at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)

at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)

at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)

at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)

at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)

at java.lang.Thread.run(Thread.java:748)

Caused by:

java.lang.NullPointerException

at org.openecomp.sdc.be.components.impl.ResponseFormatManager.getResponseFormat(ResponseFormatManager.java:61)

at org.openecomp.sdc.be.impl.ComponentsUtils.getResponseFormat(ComponentsUtils.java:160)

at org.openecomp.sdc.be.servlets.BeMonitoringServlet.getHealthCheck_aroundBody0(BeMonitoringServlet.java:104)

at org.openecomp.sdc.be.servlets.BeMonitoringServlet$AjcClosure1.run(BeMonitoringServlet.java:1)

at org.aspectj.runtime.reflect.JoinPointImpl.proceed(JoinPointImpl.java:149)

at com.jcabi.aspects.aj.MethodLogger.wrap(MethodLogger.java:212)

at com.jcabi.aspects.aj.MethodLogger.ajc$inlineAccessMethod$com_jcabi_aspects_aj_MethodLogger$com_jcabi_aspects_aj_MethodLogger$wrap(MethodLogger.java:1)

at com.jcabi.aspects.aj.MethodLogger.wrapClass(MethodLogger.java:136)

at org.openecomp.sdc.be.servlets.BeMonitoringServlet.getHealthCheck(BeMonitoringServlet.java:82)

at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

at java.lang.reflect.Method.invoke(Method.java:498)

at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)

at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144)

at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161)

at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:160)

at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99)

at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)

at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)

at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)

at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326)

at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)

at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)

at org.glassfish.jersey.internal.Errors.process(Errors.java:315)

at org.glassfish.jersey.internal.Errors.process(Errors.java:297)

 

root@ip-172-31-82-46:/dockerdata-nfs/onap/sdc/logs# cat 2017_10_11.stderrout.log

 

http://dev.onap.info:30209/logs/ete/ETE_11045/log.html#s1-s1-s1-s1-t6

 

 

00:00:00.001

KEYWORD ${session} = RequestsLibrary . Create Session asdc, ${ASDC_FE_ENDPOINT}

 

 

 

 

00:00:00.001

KEYWORD ${uuid} = UUID . Generate UUID

 

 

 

 

00:00:00.001

KEYWORD ${headers} = BuiltIn . Create Dictionary Accept=application/json, Content-Type=application/json, X-TransactionId=${GLOBAL_APPLICATION_ID}-${uuid}, X-FromAppId=${GLOBAL_APPLICATION_ID}

 

 

 

 

00:00:00.133

KEYWORD ${resp} = RequestsLibrary . Get Request asdc, ${ASDC_HEALTH_CHECK_PATH}, headers=${headers}

 

 

 

 

package org.openecomp.sdc.be.components.impl;

 

import org.openecomp.sdc.be.config.BeEcompErrorManager;

import org.openecomp.sdc.be.config.ConfigurationManager;

import org.openecomp.sdc.be.config.ErrorConfiguration;

import org.openecomp.sdc.be.config.ErrorInfo;

import org.openecomp.sdc.be.config.ErrorInfo.ErrorInfoType;

import org.openecomp.sdc.be.dao.api.ActionStatus;

import org.openecomp.sdc.common.config.EcompErrorName;

import org.openecomp.sdc.exception.OkResponseInfo;

import org.openecomp.sdc.exception.PolicyException;

import org.openecomp.sdc.exception.ResponseFormat;

import org.openecomp.sdc.exception.ServiceException;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

 

public class ResponseFormatManager {

 

private volatile static ResponseFormatManager instance;

private static ConfigurationManager configurationManager;

private static Logger log = LoggerFactory.getLogger(ResponseFormatManager.class.getName());

 

public static ResponseFormatManager getInstance() {

if (instance == null) {

 

instance = init();

}

return instance;

}

 

private static synchronized ResponseFormatManager init() {

if (instance == null) {

instance = new ResponseFormatManager();

configurationManager = ConfigurationManager.getConfigurationManager();

}

return instance;

}

 

public ResponseFormat getResponseFormat(ActionStatus responseEnum, String... variables) {

ErrorConfiguration errorConfiguration = configurationManager.getErrorConfiguration();

        ErrorInfo errorInfo = errorConfiguration.getErrorInfo(responseEnum.name());