并发环境下的Ehcache ReentrantReadWriteLock
Ehcache ReentrantReadWriteLock in concurrent environment
我在使用 Websphere 7.x 和 ehcache 2.1.0 库的生产环境中遇到问题。 Web 容器的线程都在缓存中等待查询或插入。
这是所有 webcontainer 线程挂起时的转储：
NULL
3XMTHREADINFO "WebContainer : 11" J9VMThread:0x00000000C7C10300, j9thread_t:0x0000010043913FB0, java/lang/Thread:0x00000000507623F0, state:P, prio=5
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:182)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:822) < 2 >
java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:907)
net.sf.ehcache.store.compound.Segment.put(Segment.java:402)
net.sf.ehcache.store.compound.CompoundStore.put(CompoundStore.java:132)
net.sf.ehcache.Cache.putInternal(Cache.java:1247) < 2 >
org.springframework.cache.ehcache.EhCacheCache.put(EhCacheCache.java:70)
xxx.yyy.fac.security.userdetails.GaiaLdapAuthoritiesPopulator.putElementCache(GaiaLdapAuthoritiesPopulator.java:466) < 4 >
xxx.yyy.dgtp.gaiafrontend.core.filters.preauth.GaiaGrantedAuthoritiesWebAuthenticationDetails.buildDetails(GaiaGrantedAuthoritiesWebAuthenticationDetails.java:32) < 1 >
org.springframework.security.web.authentication.preauth.AbstractPreAuthenticatedProcessingFilter.doAuthenticate(AbstractPreAuthenticatedProcessingFilter.java:114) < 1 >
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:105)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:184) < 1 >
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) < 1 >
com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:190)
com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:125)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:322)
xxx.yyy.dgtp.gaiafrontend.core.web.filters.JsonDeserializerFilter.doFilterChain(JsonDeserializerFilter.java:109) < 1 >
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
xxx.yyy.dgtp.gaiafrontend.core.filters.userinfo.UserInfoFilter.followWithTheRequestChain(UserInfoFilter.java:106) < 1 >
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
xxx.yyy.dgtp.gaia.commons.web.filters.RequestResponseWrapperFilter.doFilter(RequestResponseWrapperFilter.java:69)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:83)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
xxx.yyy.dgtp.gaiafrontend.core.web.filters.ResponseHeadersFilter.doFilter(ResponseHeadersFilter.java:55)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:184) < 1 >
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) < 1 >
com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:190)
com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:125)
org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:190)
com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:125) < 1 >
com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:908)
com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:939) < 1 >
com.ibm.ws.webcontainer.servlet.ServletWrapperImpl.handleRequest(ServletWrapperImpl.java:181)
com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:3994)
com.ibm.ws.webcontainer.webapp.WebGroup.handleRequest(WebGroup.java:276)
com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:945)
com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1592)
com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:191)
com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:453) < 3 >
com.ibm.ws.tcp.channel.impl.NewConnectionInitialReadCallback.sendToDiscriminators(NewConnectionInitialReadCallback.java:214) < 1 >
com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:175)
com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
com.ibm.io.async.AsyncChannelFuture.run(AsyncChannelFuture.java:205)
com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1660)
我使用 spring 作为 ehcache 上的抽象层，以编程方式在缓存上 get/put 的代码没有什么特别之处。
缓存的配置如下：
@Bean
public EhCacheManagerFactoryBean cacheFactoryBean() {
EhCacheManagerFactoryBean ehCacheManagerFactoryBean = new EhCacheManagerFactoryBean();
ehCacheManagerFactoryBean.setConfigLocation(new ClassPathResource("gaia-cache-ldap.xml"));
return ehCacheManagerFactoryBean;
}
@Bean
public CacheManager cacheManagerLdap() {
CacheManager cacheManager = new EhCacheCacheManager(cacheFactoryBean().getObject());
return cacheManager;
}
访问缓存的代码：
private void getMemberOfRecursive(String group, ConcurrentMap<String, String> groupsCollector) {
if (group != null) {
if (existInCache(group)) {
log.debug("Group: {} exist in cache. No query executing", group);
groupsCollector.put(group, group);
Set<String> groups = (Set<String>) getElementCache(group).get();
for (String newGroup : groups) {
getMemberOfRecursive(newGroup, groupsCollector);
}
} else {
String cn = getCnFromDn(group);
String filter = MessageFormat.format(getFilterGroupRecursive(), cn);
String baseDN = group.substring(group.indexOf(",") + 1);
groupsCollector.put(group, group);
log.debug("Executing recursive query with baseDN: {} " +
" and filter {}: ", baseDN, filter);
Set<String> groups =
getLdapTemplate().searchForSingleAttributeValues(
baseDN, filter, new String[]{}, getRetrievesAttributes());
putElementCache(group, groups);
for (String newGroup : groups) {
getMemberOfRecursive(newGroup, groupsCollector);
}
}
}
Ehcache 配置文件：
<ehcache xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="ehcache.xsd" updateCheck="false" monitoring="autodetect" dynamicConfig="true">
<diskStore path="java.io.tmpdir" />
<defaultCache maxElementsInMemory="3000" eternal="false" timeToIdleSeconds="1200"
timeToLiveSeconds="1200" overflowToDisk="true" maxElementsOnDisk="10000"
diskPersistent="false" diskExpiryThreadIntervalSeconds="120"
memoryStoreEvictionPolicy="LRU"/>
<cache name="groupsldap" maxElementsInMemory="3000" eternal="true" overflowToDisk="false"
memoryStoreEvictionPolicy="LRU"/>
</ehcache>
终于可以重现生产错误了。此错误似乎与至少存在于 Active Directory 组用户中的跨组有关。上周在没有启用缓存的情况下也跳过了这个错误，为了能够更仔细地分析它，观察到在这种情况下存在 Whosebugerror 导致 ldap 连接池耗尽，线程处于等待状态。
一旦发现是由于这个问题并包含消除以前访问过的组的修复程序，它已经在本地重现了这次测试使用缓存并且我们必须使用缓存使Web容器的线程池饱和类似于生产中检测到的行为。
我在使用 Websphere 7.x 和 ehcache 2.1.0 库的生产环境中遇到问题。 Web 容器的线程都在缓存中等待查询或插入。
这是所有 webcontainer 线程挂起时的转储：
NULL
3XMTHREADINFO "WebContainer : 11" J9VMThread:0x00000000C7C10300, j9thread_t:0x0000010043913FB0, java/lang/Thread:0x00000000507623F0, state:P, prio=5
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:182)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:822) < 2 >
java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:907)
net.sf.ehcache.store.compound.Segment.put(Segment.java:402)
net.sf.ehcache.store.compound.CompoundStore.put(CompoundStore.java:132)
net.sf.ehcache.Cache.putInternal(Cache.java:1247) < 2 >
org.springframework.cache.ehcache.EhCacheCache.put(EhCacheCache.java:70)
xxx.yyy.fac.security.userdetails.GaiaLdapAuthoritiesPopulator.putElementCache(GaiaLdapAuthoritiesPopulator.java:466) < 4 >
xxx.yyy.dgtp.gaiafrontend.core.filters.preauth.GaiaGrantedAuthoritiesWebAuthenticationDetails.buildDetails(GaiaGrantedAuthoritiesWebAuthenticationDetails.java:32) < 1 >
org.springframework.security.web.authentication.preauth.AbstractPreAuthenticatedProcessingFilter.doAuthenticate(AbstractPreAuthenticatedProcessingFilter.java:114) < 1 >
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:105)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:184) < 1 >
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) < 1 >
com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:190)
com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:125)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:322)
xxx.yyy.dgtp.gaiafrontend.core.web.filters.JsonDeserializerFilter.doFilterChain(JsonDeserializerFilter.java:109) < 1 >
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
xxx.yyy.dgtp.gaiafrontend.core.filters.userinfo.UserInfoFilter.followWithTheRequestChain(UserInfoFilter.java:106) < 1 >
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
xxx.yyy.dgtp.gaia.commons.web.filters.RequestResponseWrapperFilter.doFilter(RequestResponseWrapperFilter.java:69)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:83)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
xxx.yyy.dgtp.gaiafrontend.core.web.filters.ResponseHeadersFilter.doFilter(ResponseHeadersFilter.java:55)
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:184) < 1 >
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346) < 1 >
com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:190)
com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:125)
org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:190)
com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:125) < 1 >
com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:908)
com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:939) < 1 >
com.ibm.ws.webcontainer.servlet.ServletWrapperImpl.handleRequest(ServletWrapperImpl.java:181)
com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:3994)
com.ibm.ws.webcontainer.webapp.WebGroup.handleRequest(WebGroup.java:276)
com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:945)
com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1592)
com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:191)
com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:453) < 3 >
com.ibm.ws.tcp.channel.impl.NewConnectionInitialReadCallback.sendToDiscriminators(NewConnectionInitialReadCallback.java:214) < 1 >
com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:175)
com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
com.ibm.io.async.AsyncChannelFuture.run(AsyncChannelFuture.java:205)
com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1660)
我使用 spring 作为 ehcache 上的抽象层，以编程方式在缓存上 get/put 的代码没有什么特别之处。
缓存的配置如下：
@Bean
public EhCacheManagerFactoryBean cacheFactoryBean() {
EhCacheManagerFactoryBean ehCacheManagerFactoryBean = new EhCacheManagerFactoryBean();
ehCacheManagerFactoryBean.setConfigLocation(new ClassPathResource("gaia-cache-ldap.xml"));
return ehCacheManagerFactoryBean;
}
@Bean
public CacheManager cacheManagerLdap() {
CacheManager cacheManager = new EhCacheCacheManager(cacheFactoryBean().getObject());
return cacheManager;
}
访问缓存的代码：
private void getMemberOfRecursive(String group, ConcurrentMap<String, String> groupsCollector) {
if (group != null) {
if (existInCache(group)) {
log.debug("Group: {} exist in cache. No query executing", group);
groupsCollector.put(group, group);
Set<String> groups = (Set<String>) getElementCache(group).get();
for (String newGroup : groups) {
getMemberOfRecursive(newGroup, groupsCollector);
}
} else {
String cn = getCnFromDn(group);
String filter = MessageFormat.format(getFilterGroupRecursive(), cn);
String baseDN = group.substring(group.indexOf(",") + 1);
groupsCollector.put(group, group);
log.debug("Executing recursive query with baseDN: {} " +
" and filter {}: ", baseDN, filter);
Set<String> groups =
getLdapTemplate().searchForSingleAttributeValues(
baseDN, filter, new String[]{}, getRetrievesAttributes());
putElementCache(group, groups);
for (String newGroup : groups) {
getMemberOfRecursive(newGroup, groupsCollector);
}
}
}
Ehcache 配置文件：
<ehcache xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="ehcache.xsd" updateCheck="false" monitoring="autodetect" dynamicConfig="true">
<diskStore path="java.io.tmpdir" />
<defaultCache maxElementsInMemory="3000" eternal="false" timeToIdleSeconds="1200"
timeToLiveSeconds="1200" overflowToDisk="true" maxElementsOnDisk="10000"
diskPersistent="false" diskExpiryThreadIntervalSeconds="120"
memoryStoreEvictionPolicy="LRU"/>
<cache name="groupsldap" maxElementsInMemory="3000" eternal="true" overflowToDisk="false"
memoryStoreEvictionPolicy="LRU"/>
</ehcache>
终于可以重现生产错误了。此错误似乎与至少存在于 Active Directory 组用户中的跨组有关。上周在没有启用缓存的情况下也跳过了这个错误，为了能够更仔细地分析它，观察到在这种情况下存在 Whosebugerror 导致 ldap 连接池耗尽，线程处于等待状态。
一旦发现是由于这个问题并包含消除以前访问过的组的修复程序，它已经在本地重现了这次测试使用缓存并且我们必须使用缓存使Web容器的线程池饱和类似于生产中检测到的行为。