3549420 - Backoffice login slowness or failed due to data volume in table cvsynccronjobhistories

When you login to the backoffice. It will keep loading then get 502 Error. The login is failed.

When you check the thread dump file collected in backoffice. You could see below long running thread and hold the lock which BLOCK many other threads.

"hybrisHTTP28" #1596 daemon prio=5 os_prio=0 cpu=504123.12ms elapsed=12998.46s allocated=285G defined_classes=8 tid=0x000079547c5d0710 nid=0x84b runnable  [0x0000795212ae9000]
   java.lang.Thread.State: RUNNABLE
    at net.sf.ehcache.store.chm.SelectableConcurrentHashMap.getRandomValues(SelectableConcurrentHashMap.java:176)
    at net.sf.ehcache.store.MemoryStore.sampleElements(MemoryStore.java:653)
    at net.sf.ehcache.store.MemoryStore.findEvictionCandidate(MemoryStore.java:638)
    at net.sf.ehcache.store.MemoryStore.removeElementChosenByEvictionPolicy(MemoryStore.java:610)
    at net.sf.ehcache.store.MemoryStore.checkCapacity(MemoryStore.java:593)
    at net.sf.ehcache.store.MemoryStore.putIfAbsent(MemoryStore.java:879)
    at net.sf.ehcache.Cache.putIfAbsent(Cache.java:3744)
    at net.sf.ehcache.Cache.putIfAbsent(Cache.java:3717)
    at de.hybris.platform.regioncache.region.impl.EHCacheRegion.getWithLoader(EHCacheRegion.java:316)
    at de.hybris.platform.regioncache.DefaultCacheController.getWithLoader(DefaultCacheController.java:228)
    at de.hybris.platform.cache.impl.RegionCacheAdapter.getOrAddUnit(RegionCacheAdapter.java:213)
    at de.hybris.platform.tx.Transaction$GlobalCacheAdapter.getOrAddUnit(Transaction.java:2563)
    at de.hybris.platform.cache.AbstractCacheUnit.hintValue(AbstractCacheUnit.java:265)
    at de.hybris.platform.persistence.GenericBMPBean$GenericItemEntityStateCacheUnit.hintEntityState(GenericBMPBean.java:2036)
    at de.hybris.platform.persistence.GenericBMPBean.handleResultRow(GenericBMPBean.java:519)
    at de.hybris.platform.persistence.GenericBMPBean.handleResult(GenericBMPBean.java:496)
    at de.hybris.platform.persistence.GenericBMPBean$FindByPKListFinderResult.compute(GenericBMPBean.java:1021)
    at de.hybris.platform.cache.AbstractCacheUnit.privateGetNoLock(AbstractCacheUnit.java:297)
    at de.hybris.platform.cache.AbstractCacheUnit.privateGet(AbstractCacheUnit.java:270)
    at de.hybris.platform.cache.AbstractCacheUnit.get(AbstractCacheUnit.java:172)
    at de.hybris.platform.persistence.GenericBMPBean$FindByPKListFinderResult.getFinderResult(GenericBMPBean.java:960)
    at de.hybris.platform.persistence.GenericBMPBean.ejbFindByPKList(GenericBMPBean.java:1113)
    at jdk.internal.reflect.GeneratedMethodAccessor645.invoke(Unknown Source)
    at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.13/DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(java.base@17.0.13/Method.java:569)
    at de.hybris.platform.util.Utilities.callMethod(Utilities.java:1023)
    at de.hybris.platform.util.Utilities.callMethod(Utilities.java:1013)
    at de.hybris.platform.persistence.framework.HomeInvocationHandler.invokeMethod(HomeInvocationHandler.java:105)
    at de.hybris.platform.persistence.framework.HomeInvocationHandler.invoke(HomeInvocationHandler.java:51)
    at jdk.proxy2.$Proxy216.findByPKList(jdk.proxy2/Unknown Source)
    at de.hybris.platform.persistence.SystemEJB.findRemoteObjectsByPK(SystemEJB.java:169)
    at de.hybris.platform.core.WrapperFactory.fillFromEJB(WrapperFactory.java:978)
    at de.hybris.platform.core.WrapperFactory.fillFromPersistenceLayer(WrapperFactory.java:957)
    at de.hybris.platform.core.WrapperFactory.lambda$getItemImplForPKs$2(WrapperFactory.java:946)
    at de.hybris.platform.core.WrapperFactory$$Lambda$1167/0x0000000801686db0.accept(Unknown Source)
    at java.util.HashMap.forEach(java.base@17.0.13/HashMap.java:1421)
    at de.hybris.platform.core.WrapperFactory.getItemImplForPKs(WrapperFactory.java:946)
    at de.hybris.platform.core.WrapperFactory.getCachedItems(WrapperFactory.java:337)
    at de.hybris.platform.core.LazyLoadItemList.loadPage(LazyLoadItemList.java:220)
    at de.hybris.platform.servicelayer.search.impl.LazyLoadModelList.loadPage(LazyLoadModelList.java:98)
    at de.hybris.platform.core.LazyLoadItemList.switchPage(LazyLoadItemList.java:209)
    at de.hybris.platform.core.LazyLoadItemList.switchBufferedPageNoLock(LazyLoadItemList.java:457)
    at de.hybris.platform.core.LazyLoadItemList.switchBufferedPageSynchronized(LazyLoadItemList.java:449)
    - locked <0x0000000754eb07f0> (a de.hybris.platform.servicelayer.search.impl.LazyLoadModelList)
    at de.hybris.platform.core.LazyLoadItemList.switchBufferedPage(LazyLoadItemList.java:444)
    at de.hybris.platform.core.LazyLoadItemList.getOrSwitchBufferedPage(LazyLoadItemList.java:435)
    at de.hybris.platform.core.LazyLoadItemList.getOrSwitchBufferedPage(LazyLoadItemList.java:415)
    at de.hybris.platform.core.LazyLoadItemList.getBuffered(LazyLoadItemList.java:101)
    at de.hybris.platform.core.LazyLoadItemList.get(LazyLoadItemList.java:88)
    at de.hybris.platform.servicelayer.search.impl.LazyLoadModelList.get(LazyLoadModelList.java:117)
    at java.util.AbstractList$Itr.next(java.base@17.0.13/AbstractList.java:371)
    at de.hybris.platform.core.internal.BaseLazyLoadItemList$1.next(BaseLazyLoadItemList.java:169)
    at java.lang.Iterable.forEach(java.base@17.0.13/Iterable.java:74)
    at java.util.Collections$UnmodifiableCollection.forEach(java.base@17.0.13/Collections.java:1092)
    at com.hybris.backoffice.widgets.processes.ProcessesController.updateProcesses(ProcessesController.java:288)
    at com.hybris.backoffice.widgets.processes.ProcessesController.updateProcessForCronJob(ProcessesController.java:258)
    at com.hybris.backoffice.widgets.processes.ProcessesController.processStarted(ProcessesController.java:530)
    at jdk.internal.reflect.GeneratedMethodAccessor910.invoke(Unknown Source)
    at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.13/DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(java.base@17.0.13/Method.java:569)
    at com.hybris.cockpitng.util.WidgetControllers.lambda$bindGlobalEventListenerIfPresent$1(WidgetControllers.java:263)
    at com.hybris.cockpitng.util.WidgetControllers$$Lambda$2720/0x00000008026e0470.onEvent(Unknown Source)
    at com.hybris.cockpitng.util.CockpitEventUtils.dispatchGlobalEvents(CockpitEventUtils.java:100)
    - locked <0x0000000413208758> (a java.util.concurrent.ConcurrentHashMap)
    at com.hybris.cockpitng.events.impl.DefaultCockpitEventQueue.publishEvent(DefaultCockpitEventQueue.java:79)
    - locked <0x000000038e88e770> (a com.hybris.cockpitng.events.impl.DefaultCockpitEventQueue)
    at com.hybris.cockpitng.admin.CockpitMainWindowComposer.onMainWindowRendered(CockpitMainWindowComposer.java:242)
    at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.13/Native Method)
    at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.13/NativeMethodAccessorImpl.java:77)
    at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.13/DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(java.base@17.0.13/Method.java:569)
    at org.zkoss.zk.ui.event.GenericEventListener.onEvent(GenericEventListener.java:93)
    at org.zkoss.zk.ui.AbstractComponent.onEvent(AbstractComponent.java:3186)
    at org.zkoss.zk.ui.AbstractComponent.service(AbstractComponent.java:3156)
    at org.zkoss.zk.ui.AbstractComponent.service(AbstractComponent.java:3098)
    at org.zkoss.zk.ui.impl.EventProcessor.process(EventProcessor.java:138)
    at org.zkoss.zk.ui.impl.UiEngineImpl.processEvent(UiEngineImpl.java:1884)
    at org.zkoss.zk.ui.impl.UiEngineImpl.process(UiEngineImpl.java:1656)
    at org.zkoss.zk.ui.impl.UiEngineImpl.execUpdate(UiEngineImpl.java:1323)
    at org.zkoss.zk.au.http.DHtmlUpdateServlet.process(DHtmlUpdateServlet.java:571)
    at org.zkoss.zk.au.http.DHtmlUpdateServlet.doGet(DHtmlUpdateServlet.java:451)
    at org.zkoss.zk.au.http.DHtmlUpdateServlet.doPost(DHtmlUpdateServlet.java:459)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:555)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:623)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:209)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:51)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:218)
    at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:190)
    at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:354)
    at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:267)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at com.hybris.backoffice.mobile.filter.BackofficeMobileFilter.doFilter(BackofficeMobileFilter.java:56)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at de.hybris.platform.servicelayer.web.WebAppMediaFilter.doFilter(WebAppMediaFilter.java:161)
    at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:354)
    at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:267)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:338)
    at com.hybris.backoffice.filter.responseheaders.BackofficeLoginErrorFilter.doFilter(BackofficeLoginErrorFilter.java:33)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at com.hybris.backoffice.security.BackofficeDynamicCatalogVersionActivationFilter.doFilter(BackofficeDynamicCatalogVersionActivationFilter.java:81)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at de.hybris.platform.servicelayer.web.DataSourceSwitchingFilter.doFilter(DataSourceSwitchingFilter.java:59)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at de.hybris.platform.servicelayer.web.session.stale.impl.DefaultStaleSessionCheckingFilterChain.filterWithStaleSessionChecking(DefaultStaleSessionCheckingFilterChain.java:92)
    at de.hybris.platform.servicelayer.web.session.stale.impl.DefaultStaleSessionCheckingFilterChain.doFilter(DefaultStaleSessionCheckingFilterChain.java:58)
    at de.hybris.platform.servicelayer.web.SessionFilter.doFilterInternal(SessionFilter.java:122)
    at de.hybris.platform.servicelayer.web.SessionFilter.doFilter(SessionFilter.java:84)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at de.hybris.platform.servicelayer.web.session.HybrisSpringSessionFilter.doFilter(HybrisSpringSessionFilter.java:67)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at com.hybris.cockpitng.modules.spring.filter.ExternalModuleContextClassLoaderFilter.doFilter(ExternalModuleContextClassLoaderFilter.java:37)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at de.hybris.platform.servicelayer.web.RedirectWhenSystemIsNotInitializedFilter.doFilter(RedirectWhenSystemIsNotInitializedFilter.java:94)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at de.hybris.platform.servicelayer.web.TenantActivationFilter.doFilter(TenantActivationFilter.java:76)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at de.hybris.platform.servicelayer.web.Log4JFilter.doFilter(Log4JFilter.java:37)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at com.hybris.backoffice.filter.responseheaders.BackofficeResponseHeadersFilter.doFilter(BackofficeResponseHeadersFilter.java:31)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain$InternalFilterChain.doFilter(AbstractPlatformFilterChain.java:308)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain.processStandardFilterChain(AbstractPlatformFilterChain.java:216)
    at de.hybris.platform.servicelayer.web.AbstractPlatformFilterChain.doFilterInternal(AbstractPlatformFilterChain.java:193)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
    at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:354)
    at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:267)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at de.hybris.platform.servicelayer.web.XSSFilter.processPatternsAndDoFilter(XSSFilter.java:351)
    at de.hybris.platform.servicelayer.web.XSSFilter.doFilter(XSSFilter.java:299)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:178)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:153)
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:168)
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:90)
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:481)
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:130)
    at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:765)
    at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:670)
    at de.hybris.tomcat.StoreJvmRouteAsContextParameterValve.invoke(StoreJvmRouteAsContextParameterValve.java:44)
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:93)
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:346)
    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:390)
    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:63)
    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:928)
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1794)
    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:52)
    at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
    at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:63)
    at java.lang.Thread.run(java.base@17.0.13/Thread.java:840)

   Locked ownable synchronizers:
    - <0x0000000392507848> (a org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker)
    - <0x00000004135c81a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)

In Dynatrace, you could see the slow request like below. The rows return of table cvsynccronjobhistories is large.

SAP Commerce Cloud all version

KBA , CEC-SCC-CDM-BO-APP , Applications , Problem