We have recently experienced OutOfMemory crashes of XPages app server. The server was recently upgraded to 12.0.1FP1, but we were getting some panic crashes in HTTP even before the upgrade (it was 9.0.1FP10). Our hopes were that the upgrade would stabilize the server, but it's not the case. At least now I start to see what's the problem.
update 8.12.2022
Problem 1
We started to see OutOfMemory crash every 10-12 days and the memory was growing continuously, so I started digging. I used Eclipse MAT for the analysis as the heap dumps are still IBM-specific even when the JVM is not IBM JVM anymore. Opening of production heap dumps from the server that's up for a few days requires a few hours, so it's quite hard to do. But it showed me 2 things:
Leak detector pointed to com.sun.faces.el.impl.BeanInfoManager
The hashmap size seems to be continuously growing. There were references to classes from the apps in counts that should not be possible if the XPages apps were correctly unloaded. Further more checking the histogram view, it showed 5000 module classloaders.Isolated test 1
Isolated test 2
Verification test 3
Explanation
Solution for the BeanInfoManager problem
package mprtest;
import java.lang.reflect.Field;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import com.ibm.xsp.application.ApplicationEx;
import com.ibm.xsp.application.events.ApplicationListener2;
import com.sun.faces.el.impl.BeanInfoManager;
public class CleaningApplicationListener implements ApplicationListener2 {
public void applicationCreated(ApplicationEx arg0) {
}
public void applicationDestroyed(ApplicationEx arg0) {
ClassLoader cl=CleaningApplicationListener.class.getClassLoader();
try {
Field f = BeanInfoManager.class.getDeclaredField("mBeanInfoManagerByClass");
f.setAccessible(true);
Map m= (Map) f.get(null);
synchronized(BeanInfoManager.class) {
Iterator> it=m.entrySet().iterator();
while(it.hasNext()){
Entry me = it.next();
if (me!=null) {
Class beanInfoClass = me.getKey();
if (beanInfoClass!=null) {
ClassLoader cl2 = beanInfoClass.getClassLoader();
if (cl2!=null && cl2.equals(cl)) {
it.remove();
System.out.println("removed " + me.getKey().getName());
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
public void applicationRefreshed(ApplicationEx arg0) {
}
}
Problem 2 - SSJS
Solution to problem 2
//second part, remove the static JSContext types
try {
JSContext jscontext = JSContext.getStaticContext();
Field f = JSContext.class.getDeclaredField("registry");
f.setAccessible(true);
Registry r = (Registry) f.get(jscontext);
Field fRegistry = Registry.class.getDeclaredField("wrapperMap");
fRegistry.setAccessible(true);
IdentityHashMap<Class<?>, Object> map = (IdentityHashMap<Class<?>, Object>) fRegistry.get(r);
synchronized(r) {
Iterator<Entry<Class<?>, Object>> it=map.entrySet().iterator();
while(it.hasNext()){
Entry<Class<?>, Object> me = it.next();
if (me!=null) {
Class regClass = me.getKey();
if (regClass!=null) {
ClassLoader cl2 = regClass.getClassLoader();
if (cl2!=null && cl2.equals(cl)) {
System.out.println("removing JS type" + me.getKey().getName());
it.remove();
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
Problem 3 - Serialization
Solution to problem 3
Final version
package mprtest;
import java.lang.ref.Reference;
import java.lang.reflect.Field;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import com.ibm.jscript.JSContext;
import com.ibm.jscript.types.Registry;
import com.ibm.xsp.application.ApplicationEx;
import com.ibm.xsp.application.events.ApplicationListener2;
import com.sun.faces.el.impl.BeanInfoManager;
public class CleaningApplicationListener implements ApplicationListener2 {
public void applicationCreated(ApplicationEx arg0) {
}
public void applicationDestroyed(ApplicationEx arg0) {
ClassLoader cl=CleaningApplicationListener.class.getClassLoader();
try {
Field f = BeanInfoManager.class.getDeclaredField("mBeanInfoManagerByClass");
f.setAccessible(true);
Map<Class,BeanInfoManager> m= (Map<Class, BeanInfoManager>) f.get(null);
synchronized(BeanInfoManager.class) {
Iterator<Entry<Class, BeanInfoManager>> it=m.entrySet().iterator();
while(it.hasNext()){
Entry<Class, BeanInfoManager> me = it.next();
if (me!=null) {
Class beanInfoClass = me.getKey();
if (beanInfoClass!=null) {
ClassLoader cl2 = beanInfoClass.getClassLoader();
if (cl2!=null && cl2.equals(cl)) {
System.out.println("removed " + me.getKey().getName());
it.remove();
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
//second part, remove the static JSContext types
try {
JSContext jscontext = JSContext.getStaticContext();
Field f = JSContext.class.getDeclaredField("registry");
f.setAccessible(true);
Registry r = (Registry) f.get(jscontext);
Field fRegistry = Registry.class.getDeclaredField("wrapperMap");
fRegistry.setAccessible(true);
IdentityHashMap<Class<?>, Object> map = (IdentityHashMap<Class<?>, Object>) fRegistry.get(r);
synchronized(r) {
Iterator<Entry<Class<?>, Object>> it=map.entrySet().iterator();
while(it.hasNext()){
Entry<Class<?>, Object> me = it.next();
if (me!=null) {
Class regClass = me.getKey();
if (regClass!=null) {
ClassLoader cl2 = regClass.getClassLoader();
if (cl2!=null && cl2.equals(cl)) {
System.out.println("removing JS type" + me.getKey().getName());
it.remove();
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
//third part - serialization cache https://bugs.openjdk.org/browse/JDK-8199589
try {
Class<?> clazz = Class.forName("java.io.ObjectStreamClass$Caches");
clearCache(clazz, "localDescs", cl);
clearCache(clazz, "reflectors", cl);
} catch (Exception e) {
// Clean-up failed
e.printStackTrace();
}
}
private void clearCache(Class<?> target, String mapName, ClassLoader mycl)
throws ReflectiveOperationException, SecurityException, ClassCastException {
Field f = target.getDeclaredField(mapName);
f.setAccessible(true);
Map<?,?> map = (Map<?,?>) f.get(null);
Iterator<?> keys = map.keySet().iterator();
while (keys.hasNext()) {
Object key = keys.next();
if (key instanceof Reference) {
Object clazz = ((Reference<?>) key).get();
if (clazz instanceof Class) {
ClassLoader cl = ((Class<?>) clazz).getClassLoader();
if (cl!=null && cl.equals(mycl)) {
System.out.println("removed from ObjectStreamClass cache: "+ ((Class<?>) clazz).getName());
keys.remove();
}
}
}
}
}
public void applicationRefreshed(ApplicationEx arg0) {
}
}
Comments
Post a Comment