-
Task
-
Resolution: Done
-
Medium
-
None
-
None
-
None
-
None
20171013 - verify fix in https://gerrit.onap.org/r/#/c/18803/3
Triaging cause of HC failure - will debug line 61 to be sure
Mike mentioned an SDC chef error - looks like robot healthcheck fails on the FE container which looks ok - but the underlying call to the BE fails with a 500 - looks like there is a chef issue - looking at the actual script now
FE logs are ok
root@ip-172-31-82-46:/dockerdata-nfs/onap/sdc/logs# kubectl logs -n onap-sdc sdc-fe-1573125197-lmb2b sdc-fe
BE logs have a chef error
STDERR:
---- End output of "bash" "/tmp/jetty/chef-script20171008-1280-17vy4b8" ----
Ran "bash" "/tmp/jetty/chef-script20171008-1280-17vy4b8" returned 1
[2017-10-08T15:11:40+00:00] FATAL: Chef::Exceptions::ChildConvergeError: Chef run process exited unsuccessfully (exit code 1)
root@ip-172-31-82-46:/dockerdata-nfs/onap/sdc/logs# kubectl logs -n onap-sdc sdc-be-2986438255-9fksq sdc-be
2017-10-11 01:10:56.207:WARN:oejs.HttpChannel:qtp142638629-1532528: //sdc-be.onap-sdc:8080/sdc2/rest/healthCheck
javax.servlet.ServletException: javax.servlet.ServletException: java.lang.NullPointerException
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:139)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
Caused by:
javax.servlet.ServletException: java.lang.NullPointerException
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:489)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:427)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:388)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:341)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:228)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:840)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1772)
at org.eclipse.jetty.websocket.server.WebSocketUpgradeFilter.doFilter(WebSocketUpgradeFilter.java:205)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
Caused by:
java.lang.NullPointerException
at org.openecomp.sdc.be.components.impl.ResponseFormatManager.getResponseFormat(ResponseFormatManager.java:61)
at org.openecomp.sdc.be.impl.ComponentsUtils.getResponseFormat(ComponentsUtils.java:160)
at org.openecomp.sdc.be.servlets.BeMonitoringServlet.getHealthCheck_aroundBody0(BeMonitoringServlet.java:104)
at org.openecomp.sdc.be.servlets.BeMonitoringServlet$AjcClosure1.run(BeMonitoringServlet.java:1)
at org.aspectj.runtime.reflect.JoinPointImpl.proceed(JoinPointImpl.java:149)
at com.jcabi.aspects.aj.MethodLogger.wrap(MethodLogger.java:212)
at com.jcabi.aspects.aj.MethodLogger.ajc$inlineAccessMethod$com_jcabi_aspects_aj_MethodLogger$com_jcabi_aspects_aj_MethodLogger$wrap(MethodLogger.java:1)
at com.jcabi.aspects.aj.MethodLogger.wrapClass(MethodLogger.java:136)
at org.openecomp.sdc.be.servlets.BeMonitoringServlet.getHealthCheck(BeMonitoringServlet.java:82)
at sun.reflect.GeneratedMethodAccessor81.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:160)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
root@ip-172-31-82-46:/dockerdata-nfs/onap/sdc/logs# cat 2017_10_11.stderrout.log
http://dev.onap.info:30209/logs/ete/ETE_11045/log.html#s1-s1-s1-s1-t6
Documentation: | Runs a ASDC health check |
Start / End / Elapsed: | 20171011 10:17:51.628 / 20171011 10:17:51.769 / 00:00:00.141 |
00:00:00.001
KEYWORD ${session} = RequestsLibrary . Create Session asdc, ${ASDC_FE_ENDPOINT}
00:00:00.001
KEYWORD ${uuid} = UUID . Generate UUID
00:00:00.001
KEYWORD ${headers} = BuiltIn . Create Dictionary Accept=application/json, Content-Type=application/json, X-TransactionId=${GLOBAL_APPLICATION_ID}-${uuid}, X-FromAppId=${GLOBAL_APPLICATION_ID}
00:00:00.133
KEYWORD ${resp} = RequestsLibrary . Get Request asdc, ${ASDC_HEALTH_CHECK_PATH}, headers=${headers}
Documentation: | Send a GET request on the session object found using the | ||
Start / End / Elapsed: | 20171011 10:17:51.634 / 20171011 10:17:51.767 / 00:00:00.133 | ||
10:17:51.634 | TRACE | Arguments: [ 'asdc' | '/sdc1/rest/healthCheck' | headers={'Accept': 'application/json', 'Content-Type': 'application/json', 'X-FromAppId': 'robot-ete', 'X-TransactionId': 'robot-ete-7b230e42-3f18-4d42-a81f-9a185205572c'} ] | |
10:17:51.637 | DEBUG | Starting new HTTP connection (1): sdc-fe.onap-sdc | |
10:17:51.766 | DEBUG | http://sdc-fe.onap-sdc:8181 "GET /sdc1/rest/healthCheck HTTP/1.1" 500 660 | |
10:17:51.767 | INFO | Get Request using : alias=asdc, uri=/sdc1/rest/healthCheck, headers={u'Accept': u'application/json', u'Content-Type': u'application/json', u'X-TransactionId': u'robot-ete-7b230e42-3f18-4d42-a81f-9a185205572c', u'X-FromAppId': u'robot-ete'} | |
10:17:51.767 | TRACE | Return: <Response [500]> | |
10:17:51.767 | INFO | ${resp} = <Response [500]> |
package org.openecomp.sdc.be.components.impl;
import org.openecomp.sdc.be.config.BeEcompErrorManager;
import org.openecomp.sdc.be.config.ConfigurationManager;
import org.openecomp.sdc.be.config.ErrorConfiguration;
import org.openecomp.sdc.be.config.ErrorInfo;
import org.openecomp.sdc.be.config.ErrorInfo.ErrorInfoType;
import org.openecomp.sdc.be.dao.api.ActionStatus;
import org.openecomp.sdc.common.config.EcompErrorName;
import org.openecomp.sdc.exception.OkResponseInfo;
import org.openecomp.sdc.exception.PolicyException;
import org.openecomp.sdc.exception.ResponseFormat;
import org.openecomp.sdc.exception.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ResponseFormatManager {
private volatile static ResponseFormatManager instance;
private static ConfigurationManager configurationManager;
private static Logger log = LoggerFactory.getLogger(ResponseFormatManager.class.getName());
public static ResponseFormatManager getInstance() {
if (instance == null) {
instance = init();
}
return instance;
}
private static synchronized ResponseFormatManager init() {
if (instance == null) {
instance = new ResponseFormatManager();
configurationManager = ConfigurationManager.getConfigurationManager();
}
return instance;
}
public ResponseFormat getResponseFormat(ActionStatus responseEnum, String... variables) {
ErrorConfiguration errorConfiguration = configurationManager.getErrorConfiguration();
ErrorInfo errorInfo = errorConfiguration.getErrorInfo(responseEnum.name());