crashed TeamSite | servletd_out log.

sivaramiyer

TeamSite 671sp1 | Linux ES

1. We noticed a huge number of UtildClient processes hung in the server (close to 65-70);

2. soon the TeamSite stores went frozen

from the servletd_out log - I captured the below entries -

1. has any users ran into this before?
2. any help highly appreciated.

Thanks;

2009-04-03 11:07:23,123 ERROR com.interwoven.serverutils100.InstalledLocations - UtildClientException when calling Utility Daemon in refresh InstalledLocation: getServerProperty error
com.interwoven.serverutils100.utild.UtildClientException: getServerProperty error
at com.interwoven.serverutils100.utild.UtildClient.getServerProperty(UtildClient.java:602)
at com.interwoven.serverutils100.InstalledLocations.reset(InstalledLocations.java:170)go
at com.interwoven.serverutils100.InstalledLocations.refresh(InstalledLocations.java:154)
at com.interwoven.serverutils100.InstalledLocations.getInstance(InstalledLocations.java:138)
at com.interwoven.ui.formspub.utils.PerlPTCompiler.normalizePaths(PerlPTCompiler.java:254)
at com.interwoven.ui.formspub.utils.PerlPTCompiler.compile(PerlPTCompiler.java:130)
at com.interwoven.ui.formspub.DoGenerateCommand.execute(DoGenerateCommand.java:144)
at com.interwoven.ui.base.impl.command.CommandHandler.doExecuteCommand(CommandHandler.java:915)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandDescriptor(CommandHandler.java:752)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandID(CommandHandler.java:629)
at com.interwoven.ui.base.impl.command.CommandHandler.runCommandLoop(CommandHandler.java:234)
at com.interwoven.ui.base.impl.command.CommandServlet.doGet(CommandServlet.java:163)
at com.interwoven.ui.base.impl.command.CommandServlet.doPost(CommandServlet.java:263)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:709)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.livesite.spring.hibernate.web.filter.OpenSessionsInViewFilter.doFilterInternal(OpenSessionsInViewFilter.java:141)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:75)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:183)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:138)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.impl.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:204)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.util.SetRequestEncodingFilter.doFilter(SetRequestEncodingFilter.java:105)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:869)
at org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:664)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:80)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684)
at java.lang.Thread.run(Thread.java:595)
Caused by: com.interwoven.hopi.HopiException: Bad read
at com.interwoven.hopi.HopiContext.recvData_(HopiContext.java:910)
at com.interwoven.utild.utild_.GetServerProperty(utild_.java:2889)
at com.interwoven.utild.Utild.getServerProperty(Utild.java:449)
at com.interwoven.serverutils100.utild.UtildClient.getServerProperty(UtildClient.java:600)
... 40 more
2009-04-03 11:07:25,463 ERROR com.interwoven.serverutils100.IWConfigFileImpl - Opening connection to utility daemon has failed 2 times
2009-04-03 11:07:25,463 ERROR com.interwoven.serverutils100.IWConfigFileImpl - getFileLastModDate error
com.interwoven.serverutils100.utild.UtildClientException: getFileLastModDate error
at com.interwoven.serverutils100.utild.UtildClient.getFileLastModDate(UtildClient.java:536)
at com.interwoven.serverutils100.IWConfigFileImpl.refresh(IWConfigFileImpl.java:110)
at com.interwoven.serverutils100.IWConfig.refresh(IWConfig.java:116)
at com.interwoven.serverutils100.IWConfig.getConfig(IWConfig.java:77)
at com.interwoven.cssdk.transform.CSPTCompiler.getUMask(CSPTCompiler.java:126)
at com.interwoven.ui.formspub.utils.PerlPTCompiler.compile(PerlPTCompiler.java:185)
at com.interwoven.ui.formspub.DoGenerateCommand.execute(DoGenerateCommand.java:144)
at com.interwoven.ui.base.impl.command.CommandHandler.doExecuteCommand(CommandHandler.java:915)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandDescriptor(CommandHandler.java:752)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandID(CommandHandler.java:629)
at com.interwoven.ui.base.impl.command.CommandHandler.runCommandLoop(CommandHandler.java:234)
at com.interwoven.ui.base.impl.command.CommandServlet.doGet(CommandServlet.java:163)
at com.interwoven.ui.base.impl.command.CommandServlet.doPost(CommandServlet.java:263)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:709)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.livesite.spring.hibernate.web.filter.OpenSessionsInViewFilter.doFilterInternal(OpenSessionsInViewFilter.java:141)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:75)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:183)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:138)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.impl.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:204)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.util.SetRequestEncodingFilter.doFilter(SetRequestEncodingFilter.java:105)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:869)
at org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:664)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:80)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684)
at java.lang.Thread.run(Thread.java:595)
Caused by: com.interwoven.hopi.HopiException: Bad read
at com.interwoven.hopi.HopiContext.recvData_(HopiContext.java:910)
at com.interwoven.utild.utild_.GetFileLastModDate(utild_.java:1674)
at com.interwoven.utild.Utild.getFileLastModDate(Utild.java:397)
at com.interwoven.serverutils100.utild.UtildClient.getFileLastModDate(UtildClient.java:534)
... 40 more
2009-04-03 11:07:30,875 [http-8080-Processor8] ERROR com.interwoven.ui.formspub.datacapture.AbstractOpenDCRCommand (hmcbride commandID iw.ccpro.launch_edit_form commandID iw.ccpro.edit_form.edit_dcr.cmd0edit_dcr_cmd) - Unable to read the configuration file: templating.cfg. Please contact your system administrator.
com.interwoven.ui.formspub.datacapture.DCRPredicate$TemplatingConfigException: Couldn't instantiate TemplateManager [datacapture.error_reading_tmplcfg.error.msg ]:getFileLastModDate error
at com.interwoven.ui.formspub.datacapture.DCRPredicate.validateTemplateType(DCRPredicate.java:256)
at com.interwoven.ui.formspub.datacapture.EditDCRPredicate.execute(EditDCRPredicate.java:57)
at com.interwoven.ui.formspub.datacapture.AbstractOpenDCRCommand.execute(AbstractOpenDCRCommand.java:112)
at com.interwoven.ui.formspub.datacapture.EditDCRCommand.execute(EditDCRCommand.java:104)
at com.interwoven.ui.base.impl.command.CommandHandler.doExecuteCommand(CommandHandler.java:915)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandDescriptor(CommandHandler.java:752)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandID(CommandHandler.java:629)
at com.interwoven.ui.base.impl.command.CommandHandler.runCommand(CommandHandler.java:419)
at com.interwoven.ui.base.impl.command.CommandCallbackContextImpl.callback(CommandCallbackContextImpl.java:45)
at com.interwoven.ui.base.wizard.WizardCommand.execute(WizardCommand.java:115)
at com.interwoven.ui.base.impl.command.CommandHandler.doExecuteCommand(CommandHandler.java:915)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandDescriptor(CommandHandler.java:752)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandID(CommandHandler.java:629)
at com.interwoven.ui.base.impl.command.CommandHandler.runCommandLoop(CommandHandler.java:234)
at com.interwoven.ui.base.impl.command.CommandServlet.doGet(CommandServlet.java:163)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:689)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.livesite.spring.hibernate.web.filter.OpenSessionsInViewFilter.doFilterInternal(OpenSessionsInViewFilter.java:141)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:75)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:183)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:138)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.impl.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:204)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.util.SetRequestEncodingFilter.doFilter(SetRequestEncodingFilter.java:105)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:869)
at org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:664)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:80)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684)
at java.lang.Thread.run(Thread.java:595)
2009-04-03 11:07:55,638 ERROR com.interwoven.serverutils100.IWConfigFileImpl - Opening connection to utility daemon has failed 1 times
2009-04-03 11:07:55,638 ERROR com.interwoven.serverutils100.IWConfigFileImpl - getFileLastModDate error
com.interwoven.serverutils100.utild.UtildClientException: getFileLastModDate error
at com.interwoven.serverutils100.utild.UtildClient.getFileLastModDate(UtildClient.java:536)
at com.interwoven.serverutils100.IWConfigFileImpl.refresh(IWConfigFileImpl.java:110)
at com.interwoven.serverutils100.IWConfig.refresh(IWConfig.java:116)
at com.interwoven.serverutils100.IWConfig.getConfig(IWConfig.java:77)
at com.interwoven.ui.formspub.utils.PerlPTCompiler.compile(PerlPTCompiler.java:197)
at com.interwoven.ui.formspub.DoGenerateCommand.execute(DoGenerateCommand.java:144)
at com.interwoven.ui.base.impl.command.CommandHandler.doExecuteCommand(CommandHandler.java:915)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandDescriptor(CommandHandler.java:752)
at com.interwoven.ui.base.impl.command.CommandHandler.tryRunCommandID(CommandHandler.java:629)
at com.interwoven.ui.base.impl.command.CommandHandler.runCommandLoop(CommandHandler.java:234)
at com.interwoven.ui.base.impl.command.CommandServlet.doGet(CommandServlet.java:163)
at com.interwoven.ui.base.impl.command.CommandServlet.doPost(CommandServlet.java:263)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:709)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.livesite.spring.hibernate.web.filter.OpenSessionsInViewFilter.doFilterInternal(OpenSessionsInViewFilter.java:141)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:75)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:183)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:138)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.impl.auth.AuthenticationFilter.doFilter(AuthenticationFilter.java:204)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at com.interwoven.ui.base.util.SetRequestEncodingFilter.doFilter(SetRequestEncodingFilter.java:105)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:869)
at org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:664)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:80)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684)
at java.lang.Thread.run(Thread.java:595)
Caused by: com.interwoven.hopi.HopiException: Bad read
at com.interwoven.hopi.HopiContext.recvData_(HopiContext.java:910)
at com.interwoven.utild.utild_.GetFileLastModDate(utild_.java:1674)
at com.interwoven.utild.Utild.getFileLastModDate(Utild.java:397)
at com.interwoven.serverutils100.utild.UtildClient.getFileLastModDate(UtildClient.java:534)
... 39 more

nipper

Any updates to the logs from utild ? iwutild.log or iwutild_cmd.log (likely the former)