[xwiki-users] Lucene causing SIGSEGV and IndexReader errors

My production XWiki instance (XE 5.1) has started crashing several times a day, and I'm having some difficulty troubleshooting why - I haven't found much via searching. So far, I cannot reproduce the crash on demand, but it is happening regularly. When it crashes, catalina.out is showing either SIGSEGV errors such as the below, or occasionally just has hundreds of errors about "this IndexReader cannot be used anymore" repeated in a row. Most of them look to have something to do with Lucene - I am using Solr as the main search engine, but Lucene is still being used for search suggest. I disabled the Lucene plugin in xwiki.cfg to no effect. If these errors are indeed Lucene related, perhaps the upcoming 5.2 switch to Solr would solve this? Errors are similar to the following (condensed to save space) ***************************************************** # A fatal error has been detected by the Java Runtime Environment: # SIGSEGV (0xb) at pc=0x00007f108ea13253, pid=6446, tid=139705661753088 # JRE version: 7.0_25-b30 # Java VM: OpenJDK 64-Bit Server VM (23.7-b01 mixed mode linux-amd64 compressed oops) # Problematic frame: # J org.apache.lucene.codecs.BlockTreeTermsReader$FieldReader$SegmentTermsEnum$Frame.loadBlock()V ***************************************************** ***************************************************** # SIGSEGV (0xb) at pc=0x00007f76b669fb61, pid=20389, tid=140144995510016 # J org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsReader.visitDocument(ILorg/apache/lucene/index/StoredFieldVisitor;)V ***************************************************** ***************************************************** # SIGSEGV (0xb) at pc=0x00007f1c45178064, pid=1195, tid=139756351948544 # J org.apache.lucene.codecs.lucene40.Lucene40PostingsReader$LiveDocsSegmentDocsEnum.nextUnreadDoc()I ***************************************************** ***************************************************** # SIGSEGV (0xb) at pc=0x00007fcd11041db0, pid=25392, tid=140515950413568 # v ~StubRoutines::jbyte_disjoint_arraycopy ***************************************************** ***************************************************** 2013-09-21 17:14:23,817 [ http://www.mycompany.com/bin/get/XWiki/SuggestLuceneService?outputSyntax=pl…] ERROR c.x.x.p.l.SearchResults - Error getting search result org.apache.lucene.store.AlreadyClosedException: this IndexReader cannot be used anymore as one of its child readers was closed at org.apache.lucene.index.IndexReader.ensureOpen(IndexReader.java:257) ~[lucene-core-4.0.0.jar:4.0.0 1394950 - rmuir - 2012-10-06 03:00:40] at org.apache.lucene.index.BaseCompositeReader.document(BaseCompositeReader.java:114) ~[lucene-core-4.0.0.jar:4.0.0 1394950 - rmuir - 2012-10-06 03:00:40] at org.apache.lucene.index.IndexReader.document(IndexReader.java:436) ~[lucene-core-4.0.0.jar:4.0.0 1394950 - rmuir - 2012-10-06 03:00:40] at org.apache.lucene.search.IndexSearcher.doc(IndexSearcher.java:185) ~[lucene-core-4.0.0.jar:4.0.0 1394950 - rmuir - 2012-10-06 03:00:40] at com.xpn.xwiki.plugin.lucene.SearchResults.getRelevantResults(SearchResults.java:80) ~[xwiki-platform-search-lucene-api-5.1.jar:na] at com.xpn.xwiki.plugin.lucene.SearchResults.getHitcount(SearchResults.java:241) ~[xwiki-platform-search-lucene-api-5.1.jar:na] at com.xpn.xwiki.plugin.lucene.LucenePluginApi.getSearchResults(LucenePluginApi.java:305) ~[xwiki-platform-search-lucene-api-5.1.jar:na] at sun.reflect.GeneratedMethodAccessor680.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_25] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_25] at org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.doInvoke(UberspectImpl.java:395) ~[velocity-1.7.jar:1.7] at org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.invoke(UberspectImpl.java:384) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTMethod.execute(ASTMethod.java:173) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTReference.execute(ASTReference.java:280) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTReference.value(ASTReference.java:567) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTExpression.value(ASTExpression.java:71) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTSetDirective.render(ASTSetDirective.java:142) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.SimpleNode.render(SimpleNode.java:342) ~[velocity-1.7.jar:1.7] at org.xwiki.velocity.internal.DefaultVelocityEngine.evaluate(DefaultVelocityEngine.java:228) ~[xwiki-commons-velocity-5.1.jar:na] at org.xwiki.velocity.internal.DefaultVelocityEngine.evaluate(DefaultVelocityEngine.java:187) ~[xwiki-commons-velocity-5.1.jar:na] at org.xwiki.rendering.internal.macro.velocity.VelocityMacro.evaluateString(VelocityMacro.java:124) ~[xwiki-platform-rendering-macro-velocity-5.1.jar:na] at org.xwiki.rendering.internal.macro.velocity.VelocityMacro.evaluateString(VelocityMacro.java:50) ~[xwiki-platform-rendering-macro-velocity-5.1.jar:na] at org.xwiki.rendering.macro.script.AbstractScriptMacro.evaluateBlock(AbstractScriptMacro.java:302) ~[xwiki-platform-rendering-macro-script-5.1.jar:na] at org.xwiki.rendering.macro.script.AbstractScriptMacro.execute(AbstractScriptMacro.java:198) ~[xwiki-platform-rendering-macro-script-5.1.jar:na] at org.xwiki.rendering.macro.script.AbstractScriptMacro.execute(AbstractScriptMacro.java:59) ~[xwiki-platform-rendering-macro-script-5.1.jar:na] at org.xwiki.rendering.internal.transformation.macro.MacroTransformation.transformOnce(MacroTransformation.java:191) ~[xwiki-rendering-transformation-macro-5.1.jar:na] at org.xwiki.rendering.internal.transformation.macro.MacroTransformation.transform(MacroTransformation.java:132) ~[xwiki-rendering-transformation-macro-5.1.jar:na] at org.xwiki.rendering.internal.transformation.DefaultTransformationManager.performTransformations(DefaultTransformationManager.java:87) ~[xwiki-rendering-legacy-api-5.1.jar:na] at org.xwiki.display.internal.DocumentContentDisplayer.display(DocumentContentDisplayer.java:252) ~[xwiki-platform-display-api-5.1.jar:na] at org.xwiki.display.internal.DocumentContentDisplayer.display(DocumentContentDisplayer.java:125) ~[xwiki-platform-display-api-5.1.jar:na] at org.xwiki.display.internal.DocumentContentDisplayer.display(DocumentContentDisplayer.java:55) ~[xwiki-platform-display-api-5.1.jar:na] at org.xwiki.display.internal.DefaultDocumentDisplayer.display(DefaultDocumentDisplayer.java:80) ~[xwiki-platform-display-api-5.1.jar:na] at org.xwiki.display.internal.DefaultDocumentDisplayer.display(DefaultDocumentDisplayer.java:38) ~[xwiki-platform-display-api-5.1.jar:na] at org.xwiki.sheet.internal.SheetDocumentDisplayer.display(SheetDocumentDisplayer.java:113) ~[xwiki-platform-sheet-api-5.1.jar:na] at org.xwiki.sheet.internal.SheetDocumentDisplayer.display(SheetDocumentDisplayer.java:50) ~[xwiki-platform-sheet-api-5.1.jar:na] at org.xwiki.display.internal.ConfiguredDocumentDisplayer.display(ConfiguredDocumentDisplayer.java:67) ~[xwiki-platform-display-api-5.1.jar:na] at org.xwiki.display.internal.ConfiguredDocumentDisplayer.display(ConfiguredDocumentDisplayer.java:41) ~[xwiki-platform-display-api-5.1.jar:na] at com.xpn.xwiki.doc.XWikiDocument.getRenderedContent(XWikiDocument.java:922) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.doc.XWikiDocument.getRenderedContent(XWikiDocument.java:901) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.api.Document.getRenderedContent(Document.java:660) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at sun.reflect.GeneratedMethodAccessor674.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_25] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_25] at org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.doInvoke(UberspectImpl.java:395) ~[velocity-1.7.jar:1.7] at org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.invoke(UberspectImpl.java:384) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTMethod.execute(ASTMethod.java:173) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTReference.execute(ASTReference.java:280) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTReference.render(ASTReference.java:369) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTIfStatement.render(ASTIfStatement.java:87) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.SimpleNode.render(SimpleNode.java:342) ~[velocity-1.7.jar:1.7] at org.xwiki.velocity.internal.DefaultVelocityEngine.evaluate(DefaultVelocityEngine.java:228) ~[xwiki-commons-velocity-5.1.jar:na] at org.xwiki.velocity.internal.DefaultVelocityEngine.evaluate(DefaultVelocityEngine.java:187) ~[xwiki-commons-velocity-5.1.jar:na] at com.xpn.xwiki.render.XWikiVelocityRenderer.evaluate(XWikiVelocityRenderer.java:105) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.internal.template.DefaultPrivilegedTemplateRenderer.evaluate(DefaultPrivilegedTemplateRenderer.java:125) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.internal.template.DefaultPrivilegedTemplateRenderer.evaluateTemplate(DefaultPrivilegedTemplateRenderer.java:75) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.XWiki.evaluateTemplate(XWiki.java:1829) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.XWiki.parseTemplate(XWiki.java:1769) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.api.XWiki.parseTemplate(XWiki.java:895) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at sun.reflect.GeneratedMethodAccessor171.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_25] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_25] at org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.doInvoke(UberspectImpl.java:395) ~[velocity-1.7.jar:1.7] at org.apache.velocity.util.introspection.UberspectImpl$VelMethodImpl.invoke(UberspectImpl.java:384) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTMethod.execute(ASTMethod.java:173) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTReference.execute(ASTReference.java:280) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTReference.render(ASTReference.java:369) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTBlock.render(ASTBlock.java:72) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.directive.VelocimacroProxy.render(VelocimacroProxy.java:216) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.directive.RuntimeMacro.render(RuntimeMacro.java:311) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.directive.RuntimeMacro.render(RuntimeMacro.java:230) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.ASTDirective.render(ASTDirective.java:207) ~[velocity-1.7.jar:1.7] at org.apache.velocity.runtime.parser.node.SimpleNode.render(SimpleNode.java:342) ~[velocity-1.7.jar:1.7] at org.xwiki.velocity.internal.DefaultVelocityEngine.evaluate(DefaultVelocityEngine.java:228) ~[xwiki-commons-velocity-5.1.jar:na] at org.xwiki.velocity.internal.DefaultVelocityEngine.evaluate(DefaultVelocityEngine.java:187) ~[xwiki-commons-velocity-5.1.jar:na] at com.xpn.xwiki.render.XWikiVelocityRenderer.evaluate(XWikiVelocityRenderer.java:105) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.internal.template.DefaultPrivilegedTemplateRenderer.evaluate(DefaultPrivilegedTemplateRenderer.java:125) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.internal.template.DefaultPrivilegedTemplateRenderer.evaluateTemplate(DefaultPrivilegedTemplateRenderer.java:75) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.XWiki.evaluateTemplate(XWiki.java:1829) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.web.Utils.parseTemplate(Utils.java:166) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.web.XWikiAction.execute(XWikiAction.java:295) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at com.xpn.xwiki.web.XWikiAction.execute(XWikiAction.java:128) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at org.apache.struts.action.RequestProcessor.processActionPerform(RequestProcessor.java:431) ~[struts-1.2.9.jar:1.2.9] at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:236) ~[struts-1.2.9.jar:1.2.9] at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1196) ~[struts-1.2.9.jar:1.2.9] at org.apache.struts.action.ActionServlet.doGet(ActionServlet.java:414) ~[struts-1.2.9.jar:1.2.9] at javax.servlet.http.HttpServlet.service(HttpServlet.java:621) ~[tomcat-servlet-api-3.0.jar:na] at javax.servlet.http.HttpServlet.service(HttpServlet.java:728) ~[tomcat-servlet-api-3.0.jar:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) ~[tomcat-catalina-7.0.35.jar:7.0.35] at com.xpn.xwiki.web.ActionFilter.doFilter(ActionFilter.java:120) ~[xwiki-platform-legacy-oldcore-5.1.jar:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.xwiki.wysiwyg.server.filter.ConversionFilter.doFilter(ConversionFilter.java:144) ~[xwiki-platform-wysiwyg-server-5.1.jar:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) ~[tomcat-catalina-7.0.35.jar:7.0.35] at com.xpn.xwiki.plugin.webdav.XWikiDavFilter.doFilter(XWikiDavFilter.java:66) ~[xwiki-platform-webdav-server-5.1.jar:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.xwiki.container.servlet.filters.internal.SavedRequestRestorerFilter.doFilter(SavedRequestRestorerFilter.java:208) ~[xwiki-platform-container-servlet-5.1.jar:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.xwiki.container.servlet.filters.internal.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:111) ~[xwiki-platform-container-servlet-5.1.jar:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:222) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:123) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:171) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:99) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:936) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:407) ~[tomcat-catalina-7.0.35.jar:7.0.35] at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1004) ~[tomcat-coyote-7.0.35.jar:7.0.35] at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:589) ~[tomcat-coyote-7.0.35.jar:7.0.35] at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:312) ~[tomcat-coyote-7.0.35.jar:7.0.35] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_25] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.7.0_25] at java.lang.Thread.run(Thread.java:724) ~[na:1.7.0_25] ***************************************************** Thanks, aaron