记录一次tomcat线程阻塞引发的系统卡死、获取不到session

Mars_xiaotao

2121人浏览 · 2021-04-28 18:54:25

Mars_xiaotao · 2021-04-28 18:54:25 发布

项目背景：该项目是事业单位内部办公系统，两三百人的用户量，应用服务器（windowServer2012）系统盘空间时常不足（经常远程清理）。该系统生产环境也时常卡顿，之前一直以为是应用服务器系统盘空间不足导致。
后来在一次次强调下，客户终于对服务器进行了优化，本以为一切都好了。
但是一个明媚的上午，系统再次卡死，客户心态爆炸，领导严令立即远程排查mysql数据库、tomcat中间件是否进行过优化。屋漏偏逢连雨天啊，那天我起晚了，9点多了还没到公司…… 一路飞奔到公司。

mysql数据库的优化是我调整的我是知道的，tomcat在项目部署之初实施人员应该调整过的，远程查看了也确实如此。

查看报错日志如下： IO异常，服务器主机拒接连接

org.apache.catalina.connector.ClientAbortException: java.io.IOException
	at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:407) ~[catalina.jar:7.0.69]
	at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:371) ~[tomcat-coyote.jar:7.0.69]
	at org.apache.catalina.connector.OutputBuffer.writeBytes(OutputBuffer.java:432) ~[catalina.jar:7.0.69]
	at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:420) ~[catalina.jar:7.0.69]
	at org.apache.catalina.connector.CoyoteOutputStream.write(CoyoteOutputStream.java:91) ~[catalina.jar:7.0.69]
	at org.apache.catalina.servlets.DefaultServlet.copy(DefaultServlet.java:2080) ~[catalina.jar:7.0.69]
	at org.apache.catalina.servlets.DefaultServlet.serveResource(DefaultServlet.java:1026) ~[catalina.jar:7.0.69]
	at org.apache.catalina.servlets.DefaultServlet.doGet(DefaultServlet.java:441) ~[catalina.jar:7.0.69]
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:624) [servlet-api.jar:na]
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) [servlet-api.jar:na]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:747) ~[catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:485) ~[catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:377) ~[catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:337) ~[catalina.jar:7.0.69]
	at org.springframework.web.servlet.resource.DefaultServletHttpRequestHandler.handleRequest(DefaultServletHttpRequestHandler.java:122) ~[spring-webmvc-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.servlet.mvc.HttpRequestHandlerAdapter.handle(HttpRequestHandlerAdapter.java:51) ~[spring-webmvc-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:945) [spring-webmvc-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:876) [spring-webmvc-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:931) [spring-webmvc-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:822) [spring-webmvc-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:624) [servlet-api.jar:na]
	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:807) [spring-webmvc-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) [servlet-api.jar:na]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) [tomcat7-websocket.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at com.alibaba.druid.support.http.WebStatFilter.doFilter(WebStatFilter.java:83) [druid-1.0.11.jar:1.0.11]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at org.activiti.explorer.servlet.JsonpCallbackFilter.doFilter(JsonpCallbackFilter.java:57) [activiti-explorer-servlet-5.21.0.jar:na]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at org.springframework.orm.jpa.support.OpenEntityManagerInViewFilter.doFilterInternal(OpenEntityManagerInViewFilter.java:177) [spring-orm-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:108) [spring-web-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88) [spring-web-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:108) [spring-web-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at com.yonyou.cap.isc.shiro.filter.SafeAuthcFilter.doFilter(SafeAuthcFilter.java:170) [cap-isc-2.0.1-SNAPSHOT.jar:na]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449) [shiro-web-1.2.3.1-SNAPSHOT.jar:na]
	at org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365) [shiro-web-1.2.3.1-SNAPSHOT.jar:na]
	at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90) [shiro-core-1.2.3.1-SNAPSHOT.jar:1.2.3.1-SNAPSHOT]
	at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83) [shiro-core-1.2.3.1-SNAPSHOT.jar:1.2.3.1-SNAPSHOT]
	at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383) [shiro-core-1.2.3.1-SNAPSHOT.jar:1.2.3.1-SNAPSHOT]
	at org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362) [shiro-web-1.2.3.1-SNAPSHOT.jar:na]
	at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125) [shiro-web-1.2.3.1-SNAPSHOT.jar:na]
	at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:344) [spring-web-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:261) [spring-web-4.0.0.RELEASE.jar:4.0.0.RELEASE]
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) [catalina.jar:7.0.69]
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) [catalina.jar:7.0.69]
	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) [catalina.jar:7.0.69]
	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) [catalina.jar:7.0.69]
	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505) [catalina.jar:7.0.69]
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169) [catalina.jar:7.0.69]
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) [catalina.jar:7.0.69]
	at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956) [catalina.jar:7.0.69]
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) [catalina.jar:7.0.69]
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436) [catalina.jar:7.0.69]
	at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078) [tomcat-coyote.jar:7.0.69]
	at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625) [tomcat-coyote.jar:7.0.69]
	at org.apache.tomcat.util.net.AprEndpoint$SocketProcessor.doRun(AprEndpoint.java:2517) [tomcat-coyote.jar:7.0.69]
	at org.apache.tomcat.util.net.AprEndpoint$SocketProcessor.run(AprEndpoint.java:2506) [tomcat-coyote.jar:7.0.69]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_75]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_75]
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) [tomcat-coyote.jar:7.0.69]
	at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75]
Caused by: java.io.IOException: null
	at org.apache.coyote.http11.InternalAprOutputBuffer.flushBuffer(InternalAprOutputBuffer.java:205) ~[tomcat-coyote.jar:7.0.69]
	at org.apache.coyote.http11.InternalAprOutputBuffer.access$100(InternalAprOutputBuffer.java:37) ~[tomcat-coyote.jar:7.0.69]
	at org.apache.coyote.http11.InternalAprOutputBuffer$SocketOutputBuffer.doWrite(InternalAprOutputBuffer.java:235) ~[tomcat-coyote.jar:7.0.69]
	at org.apache.coyote.http11.filters.IdentityOutputFilter.doWrite(IdentityOutputFilter.java:84) ~[tomcat-coyote.jar:7.0.69]
	at org.apache.coyote.http11.AbstractOutputBuffer.doWrite(AbstractOutputBuffer.java:192) ~[tomcat-coyote.jar:7.0.69]
	at org.apache.coyote.Response.doWrite(Response.java:499) ~[tomcat-coyote.jar:7.0.69]
	at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:402) ~[catalina.jar:7.0.69]
	... 73 common frames omitted

是什么原因造成的呢？登录不上，是不是session没释放呢？
于是去查看shiro上下文配置，对会话超时等属性也做了配置。
在这里插入图片描述
同事说，应该还是tomcat的问题，tomcat默认是阻塞协议，server.xml中添加如下非阻塞协议配置。

此处注意，tomcat7是org.apache.coyote.http11.Http11NioProtocol
tomcat8是org.apache.coyote.http11.Http11Nio2Protocol