I inherited support of an application that runs
Tomcat 4.1.18 on Server 2003. It was put into production back in Dec of 2005. During peak usage (on Fridays), Tomcat builds memory usage up to around 1,000,000K and then crashes and restarts. The vendor for my web application supports Tomcat to some extent, but they have not been able to tell me what the problem is. We played with the Catalina_opts variables, but no luck. They are trying to tell me that Tomcat is not capable of the load we are putting on it. I find that hard to believe, since I can look back in the event viewer logs and see that Tomcat has crashed multiple times every Friday since it was put into production.
Here is a little more information about what I am seeing. I am sure I will need to post more, so just let me know what. Thanks to anyone that takes a look at this. I took a lookt at the FAQ for Java.lang.OutofMemory, but didn't not see anything specific to the native
thread message.
set CATALINA_OPTS=-Xms1024m -Xmx1024m -XX:MaxPermSize=256m -server -XX:-UseOnStackReplacement -Djava.awt.headless=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:f:\sashome\EntBIServer\Lev1\web\Deployments\Portal\logs\garbage
2008-08-01 08:16:43 StandardWrapperValve[action]: Servlet.service() for
servlet action threw exception
----- Root Cause -----
java.lang.OutOfMemoryError: unable to create new native thread
at java.lang.Thread.start(Native Method)
at sun.rmi.transport.tcp.TCPChannel.free(TCPChannel.java:322)
at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:157)
at com.sas.services.session.SessionService_Stub.getServiceState(Unknown Source)
at com.sas.services.ServiceProxy.getServiceState(ServiceProxy.java:454)
at com.sas.services.discovery.ServicesCache.getServices(ServicesCache.java:120)
at com.sas.services.discovery.DiscoveryService.findInCache(DiscoveryService.java:961)
at com.sas.services.discovery.DiscoveryService.findServicesInCache(DiscoveryService.java:716)
at com.sas.services.discovery.DiscoveryService.findServices(DiscoveryService.java:627)
at com.sas.services.discovery.DiscoveryService.findServices(DiscoveryService.java:524)
at com.sas.services.discovery.AbstractDiscoveryService.findService(AbstractDiscoveryService.java:81)
at com.sas.services.webapp.ServicesFacade.getRemoteServiceInterface(ServicesFacade.java:292)
at com.sas.services.webapp.ServicesFacade.getRemoteSessionService(ServicesFacade.java:197)
at com.sas.portal.delegates.authentication.AbstractLogonDelegate.isQuiesced(AbstractLogonDelegate.java:372)
at com.sas.portal.delegates.authentication.LogonDelegate.logonPublic(LogonDelegate.java:56)
at com.sas.portal.delegates.authentication.AuthenticationFacade.logonPublic(AuthenticationFacade.java:72)
at com.sas.portal.httpfilters.SecurityFilter.doFilter(SecurityFilter.java:106)
at com.sas.portal.httpfilters.HttpFilter.doFilter(HttpFilter.java:40)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:213)
at org.apache.catalina.core.ApplicationFilterChain.access$000(ApplicationFilterChain.java:98)
at org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:176)
at java.security.AccessController.doPrivileged(Native Method)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:172)
at com.sas.services.webapp.lifecycle.ObjectTrackerFilter.doFilter(ObjectTrackerFilter.java:56)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:213)
at org.apache.catalina.core.ApplicationFilterChain.access$000(ApplicationFilterChain.java:98)
at org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:176)
at java.security.AccessController.doPrivileged(Native Method)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:172)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:260)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardContext.invoke(StandardContext.java:2415)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:180)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.valves.ErrorDispatcherValve.invoke(ErrorDispatcherValve.java:170)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:641)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:172)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:641)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:174)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.coyote.tomcat4.CoyoteAdapter.service(CoyoteAdapter.java:223)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:432)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:386)
at org.apache.tomcat.util.net.TcpWorkerThread.runIt(PoolTcpEndpoint.java:534)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:530)
at java.lang.Thread.run(Thread.java:534)
2008-08-01 08:38:38 StandardWrapperValve[
jsp]: Servlet.service() for servlet jsp threw exception
org.apache.jasper.JasperException: unable to create new native thread
at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:248)
at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:295)
at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:241)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:853)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:247)
at org.apache.catalina.core.ApplicationFilterChain.access$000(ApplicationFilterChain.java:98)
at org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:176)
at java.security.AccessController.doPrivileged(Native Method)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:172)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:260)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardContext.invoke(StandardContext.java:2415)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:180)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.valves.ErrorDispatcherValve.invoke(ErrorDispatcherValve.java:170)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:641)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:172)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:641)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:174)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.coyote.tomcat4.CoyoteAdapter.service(CoyoteAdapter.java:223)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:432)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:386)
at org.apache.tomcat.util.net.TcpWorkerThread.runIt(PoolTcpEndpoint.java:534)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:530)
at java.lang.Thread.run(Thread.java:534)
----- Root Cause -----
javax.servlet.ServletException: unable to create new native thread
at org.apache.jasper.runtime.PageContextImpl.handlePageException(PageContextImpl.java:533)
at org.apache.jsp.detailData_jsp._jspService(detailData_jsp.java:378)
at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:137)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:853)
at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:204)
at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:295)
at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:241)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:853)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:247)
at org.apache.catalina.core.ApplicationFilterChain.access$000(ApplicationFilterChain.java:98)
at org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:176)
at java.security.AccessController.doPrivileged(Native Method)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:172)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:260)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardContext.invoke(StandardContext.java:2415)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:180)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.valves.ErrorDispatcherValve.invoke(ErrorDispatcherValve.java:170)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:641)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:172)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:641)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:174)
at org.apache.catalina.core.StandardPipeline$StandardPipelineValveContext.invokeNext(StandardPipeline.java:643)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:480)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:995)
at org.apache.coyote.tomcat4.CoyoteAdapter.service(CoyoteAdapter.java:223)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:432)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:386)
at org.apache.tomcat.util.net.TcpWorkerThread.runIt(PoolTcpEndpoint.java:534)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:530)
at java.lang.Thread.run(Thread.java:534)