Spring Data Neo4j 超出了 GC 开销限制

Question

问题： 我正在尝试将 DBLP 数据库（从公开可用的 XML file）导入带有 Spring 应用程序的 Neo4j 实例，但是在该应用程序已将大约 39 000 份出版物导入 Neo4j 数据库，我收到一条 "GC overhead limit exceeded" 消息。

注意：我已经在whosebug.com上彻底寻找答案，但即使这里有很多好的答案，我也无法解决问题。

这里是入口点Application.java:

package hello;

import java.io.File;

import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.kernel.impl.util.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.neo4j.config.EnableNeo4jRepositories;
import org.springframework.data.neo4j.config.Neo4jConfiguration;
import org.springframework.data.neo4j.core.GraphDatabase;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.xml.sax.SAXException;
import org.apache.xerces.util.SecurityManager;

/**
 * 
 * Inspiration: http://spring.io/guides/gs/accessing-data-neo4j/
 *
 */
@SpringBootApplication
public class Application implements CommandLineRunner {

    @Configuration
    @EnableNeo4jRepositories(basePackages = "hello")
    static class ApplicationConfig extends Neo4jConfiguration {

        public ApplicationConfig() {
            setBasePackage("hello");
        }

        @Bean
        GraphDatabaseService graphDatabaseService() {
            return new GraphDatabaseFactory().newEmbeddedDatabase("accessingdataneo4j.db");
        }

    }

    @Autowired
    PublicationRepository publicationRepository;

    @Autowired
    GraphDatabase graphDatabase;

    public void run(String... args) throws Exception {

        Transaction tx = graphDatabase.beginTx();

        try {

            SAXParserFactory parserFactory = SAXParserFactory.newInstance();
            SAXParser parser = parserFactory.newSAXParser();

            SecurityManager mgr = new SecurityManager();
            mgr.setEntityExpansionLimit(3100000);
            parser.setProperty("http://apache.org/xml/properties/security-manager", mgr);

            SaxHandler handler = new SaxHandler(publicationRepository);
            parser.getXMLReader().setFeature("http://xml.org/sax/features/validation", true);
            InputStream xmlInput = new FileInputStream("/Users/username/Downloads/dblp.xml");
            parser.parse(xmlInput, handler);

            tx.success();

        } catch (SAXException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (ParserConfigurationException e) {
            e.printStackTrace();
        } finally {
            tx.close();
        }

    }

    public static void main(String[] args) throws Exception {
        FileUtils.deleteRecursively(new File("accessingdataneo4j.db"));
        SpringApplication.run(Application.class, args);
    }

}

这是 Application.java 使用的 SAX 处理程序：

package hello;

import java.util.Stack;

import org.springframework.beans.factory.annotation.Autowired;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

public class SaxHandler extends DefaultHandler {

    private Stack<String> elementStack = new Stack<String>();
    private Stack<Publication> objectStack = new Stack<Publication>();
    private String publicationType = null;
    private PublicationRepository publicationRepository = null;
    private Publication publication = null;
    private Author author = null;
    private String currentElement = null;
    private String value = null;

    private static int counter = 0;

    @Autowired
    public SaxHandler(PublicationRepository publicationRepository) {
        this.publicationRepository = publicationRepository;
    }

    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

        elementStack.push(qName);

        publication = new Publication();

        if ("article".equals(qName)) {
            publication.setType("article");
        } else if ("inproceedings".equals(qName)) {
            publication.setType("inproceedings");
        } else if ("proceedings".equals(qName)) {
            publication.setType("proceedings");
        } else if ("book".equals(qName)) {
            publication.setType("book");
        } else if ("incollection".equals(qName)) {
            publication.setType("incollection");
        } else if ("phdthesis".equals(qName)) {
            publication.setType("phdthesis");
        } else if ("mastersthesis".equals(qName)) {
            publication.setType("mastersthesis");
        } else if ("www".equals(qName)) {
            publication.setType("www");
        }

        if (attributes.getLength() > 0) {           
            publicationType = qName;

            if (attributes.getValue("key") != null) {
                publication.setKey(attributes.getValue("key"));
            }
            if (attributes.getValue("mdate") != null) {
                publication.setMdate(attributes.getValue("mdate"));
            }
            if (attributes.getValue("publtype") != null) {
                publication.setMdate(attributes.getValue("publtype"));
            }
            if (attributes.getValue("reviewid") != null) {
                publication.setMdate(attributes.getValue("reviewid"));
            }
            if (attributes.getValue("rating") != null) {
                publication.setMdate(attributes.getValue("rating"));
            }

            objectStack.push(publication);
        }
    }

    public void endElement(String uri, String localName, String qName) throws SAXException {

        elementStack.pop();

        if (publicationType.equals(qName)) {
            publicationRepository.save(objectStack.pop());
            ++counter;
            if (counter % 1000 == 0) {
                System.out.println("counter = " + counter);
                System.out.println("element stack size = " + elementStack.size());
                System.out.println("object stack size = " + objectStack.size());
                for (Publication p : objectStack) {
                    System.out.println("objectStack: " + p);
                }
            }
            if (counter % 5000 == 0) {
                System.gc();
            }
        }

    }

    public void characters(char ch[], int start, int length) throws SAXException {

        value = new String(ch,start,length).trim();

        if (value.length() == 0)
            return;

        publication = objectStack.peek();
        currentElement = elementStack.peek();

        if ("author".equals(currentElement)) {          
            author = new Author();
            author.setName(value);
            publication.addAuthor(author);
        } else if ("editor".equals(currentElement)) {
            publication.setEditor(value);
        } else if ("title".equals(currentElement)) {
            publication.setTitle(value);
        } else if ("booktitle".equals(currentElement)) {
            publication.setBooktitle(value);
        } else if ("pages".equals(currentElement)) {
            publication.setPages(value);
        } else if ("year".equals(currentElement)) {
            publication.setYear(value);
        } else if ("address".equals(currentElement)) {
            publication.setAddress(value);
        } else if ("journal".equals(currentElement)) {
            publication.setJournal(value);
        } else if ("volume".equals(currentElement)) {
            publication.setVolume(value);
        } else if ("number".equals(currentElement)) {
            publication.setNumber(value);
        } else if ("month".equals(currentElement)) {
            publication.setMonth(value);
        } else if ("url".equals(currentElement)) {
            publication.setUrl(value);
        } else if ("ee".equals(currentElement)) {
            publication.setEe(value);
        } else if ("cdrom".equals(currentElement)) {
            publication.setCdrom(value);
        } else if ("cite".equals(currentElement)) {
            publication.setCite(value);
        } else if ("publisher".equals(currentElement)) {
            publication.setPublisher(value);
        } else if ("note".equals(currentElement)) {
            publication.setNote(value);
        } else if ("crossref".equals(currentElement)) {
            publication.setCrossref(value);
        } else if ("isbn".equals(currentElement)) {
            publication.setIsbn(value);
        } else if ("series".equals(currentElement)) {
            publication.setSeries(value);
        } else if ("school".equals(currentElement)) {
            publication.setSchool(value);
        } else if ("chapter".equals(currentElement)) {
            publication.setChapter(value);
        }
    }
}

这里是PublicationRepository.java:

package hello;

import org.springframework.data.repository.CrudRepository;

public interface PublicationRepository extends CrudRepository<Publication, String> {

    Publication findByTitle(String title);

}

这里是Author.java，领域模型之一：

package hello;

import org.springframework.data.neo4j.annotation.GraphId;
import org.springframework.data.neo4j.annotation.NodeEntity;

@NodeEntity
public class Author {

    @GraphId
    private Long id;

    private String name;

    public Author() {

    }

    public Author(String name) {

    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;

        if (obj == null)
            return false;

        if (this.getClass() != obj.getClass())
            return false;

        Author other = (Author) obj;

        if (this.id != null && this.name != null && other.id != null && other.name != null) {
            if (this.id.equals(other.id) && this.name.equals(other.name))
                return true;
        } else {
            return true;
        }

        return false;
    }

    @Override
    public int hashCode() {
        return 31 * (this.id == null ? 1 : this.id.hashCode()) + 31 * (this.name == null ? 1 : this.name.hashCode());
    }

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }
}

这里是Publication.java，第二个领域模型：

package hello;

import java.io.Serializable;
import java.util.HashSet;
import java.util.Set;

import org.neo4j.graphdb.Direction;
import org.springframework.data.neo4j.annotation.GraphId;
import org.springframework.data.neo4j.annotation.NodeEntity;
import org.springframework.data.neo4j.annotation.RelatedTo;

@NodeEntity
public class Publication implements Serializable {

    private static final long serialVersionUID = -6393545300391560520L;

    @GraphId 
    Long nodeId;

    private String type = "";
    private String key = "";
    private String mdate = "";
    private String publtype = "";
    private String reviewid = "";
    private String rating = "";

    @RelatedTo(type = "WROTE", direction = Direction.INCOMING)
    private Set<Author> authors = new HashSet<Author>();
    private String editor = "";
    private String title = "";
    private String booktitle = "";
    private String pages = "";
    private String year = "";
    private String address = "";
    private String journal = "";
    private String volume = "";
    private String number = "";
    private String month = "";
    private String url = "";
    private String ee = "";
    private String cdrom = "";
    private String cite = "";
    private String publisher = "";
    private String note = "";
    private String crossref = "";
    private String isbn = "";
    private String series = "";
    private String school = "";
    private String chapter = "";

    public Publication() {

    }

    public void addAuthor(Author author) {
        authors.add(author);
    }

    public Set<Author> getAuthors() {
        return authors;
    }

    public void setAuthors(Set<Author> authors) {
        this.authors = authors;
    }

    @Override
    public String toString() {
        return "TYPE: " + type + "\n" 
                + "KEY: " + key + "\n"
                + "MDATE: " + mdate + "\n";
    }

    public Long getNodeId() {
        return nodeId;
    }

    public void setNodeId(Long nodeId) {
        this.nodeId = nodeId;
    }

    public String getKey() {
        return key;
    }

    public void setKey(String key) {
        this.key = key;
    }

    public String getMdate() {
        return mdate;
    }

    public void setMdate(String mdate) {
        this.mdate = mdate;
    }

    public String getPubltype() {
        return publtype;
    }

    public void setPubltype(String publtype) {
        this.publtype = publtype;
    }

    public String getReviewid() {
        return reviewid;
    }

    public void setReviewid(String reviewid) {
        this.reviewid = reviewid;
    }

    public String getRating() {
        return rating;
    }

    public void setRating(String rating) {
        this.rating = rating;
    }

    public String getType() {
        return type;
    }

    public void setType(String type) {
        this.type = type;
    }

    public String getEditor() {
        return editor;
    }

    public void setEditor(String editor) {
        this.editor = editor;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String getBooktitle() {
        return booktitle;
    }

    public void setBooktitle(String booktitle) {
        this.booktitle = booktitle;
    }

    public String getPages() {
        return pages;
    }

    public void setPages(String pages) {
        this.pages = pages;
    }

    public String getYear() {
        return year;
    }

    public void setYear(String year) {
        this.year = year;
    }

    public String getAddress() {
        return address;
    }

    public void setAddress(String address) {
        this.address = address;
    }

    public String getJournal() {
        return journal;
    }

    public void setJournal(String journal) {
        this.journal = journal;
    }

    public String getVolume() {
        return volume;
    }

    public void setVolume(String volume) {
        this.volume = volume;
    }

    public String getNumber() {
        return number;
    }

    public void setNumber(String number) {
        this.number = number;
    }

    public String getMonth() {
        return month;
    }

    public void setMonth(String month) {
        this.month = month;
    }

    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }

    public String getEe() {
        return ee;
    }

    public void setEe(String ee) {
        this.ee = ee;
    }

    public String getCdrom() {
        return cdrom;
    }

    public void setCdrom(String cdrom) {
        this.cdrom = cdrom;
    }

    public String getCite() {
        return cite;
    }

    public void setCite(String cite) {
        this.cite = cite;
    }

    public String getPublisher() {
        return publisher;
    }

    public void setPublisher(String publisher) {
        this.publisher = publisher;
    }

    public String getNote() {
        return note;
    }

    public void setNote(String note) {
        this.note = note;
    }

    public String getCrossref() {
        return crossref;
    }

    public void setCrossref(String crossref) {
        this.crossref = crossref;
    }

    public String getIsbn() {
        return isbn;
    }

    public void setIsbn(String isbn) {
        this.isbn = isbn;
    }

    public String getSeries() {
        return series;
    }

    public void setSeries(String series) {
        this.series = series;
    }

    public String getSchool() {
        return school;
    }

    public void setSchool(String school) {
        this.school = school;
    }

    public String getChapter() {
        return chapter;
    }

    public void setChapter(String chapter) {
        this.chapter = chapter;
    }

}

该应用程序使用 Maven 作为构建自动化工具，这里是 pom.xml:

<?xml version="1.0" encoding="UTF-8"?>
<project    xmlns="http://maven.apache.org/POM/4.0.0" 
            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
            xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>com.dblp</groupId>
    <artifactId>graphdbcreator</artifactId>
    <version>0.1.0</version>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>1.2.2.RELEASE</version>
    </parent>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-context</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-tx</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.data</groupId>
            <artifactId>spring-data-neo4j</artifactId>
        </dependency>
        <dependency>
            <groupId>org.hibernate</groupId>
            <artifactId>hibernate-validator</artifactId>
        </dependency>
        <dependency>
            <groupId>javax.el</groupId>
            <artifactId>javax.el-api</artifactId>
            <version>2.2.4</version>
        </dependency>
        <dependency>
            <groupId>xerces</groupId>
            <artifactId>xercesImpl</artifactId>
            <version>2.8.0</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

    <repositories>
        <repository>
            <id>spring-releases</id>
            <name>Spring Releases</name>
            <url>https://repo.spring.io/libs-release</url>
        </repository>
        <repository>
            <id>neo4j</id>
            <name>Neo4j</name>
            <url>http://m2.neo4j.org/</url>
        </repository>
    </repositories>

</project>

这是错误消息的中心部分：

java.lang.OutOfMemoryError: GC overhead limit exceeded
    at org.neo4j.collection.primitive.PrimitiveIntCollections.toPrimitiveIterator(PrimitiveIntCollections.java:678)
    at org.neo4j.kernel.impl.api.StateHandlingStatementOperations.nodeGetLabels(StateHandlingStatementOperations.java:188)
    at org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations.nodeSetProperty(ConstraintEnforcingEntityOperations.java:85)
    at org.neo4j.kernel.impl.api.LockingStatementOperations.nodeSetProperty(LockingStatementOperations.java:280)
    at org.neo4j.kernel.impl.api.OperationsFacade.nodeSetProperty(OperationsFacade.java:551)
    at org.neo4j.kernel.impl.core.NodeProxy.setProperty(NodeProxy.java:254)
    at org.springframework.data.neo4j.fieldaccess.PropertyFieldAccessorFactory$PropertyFieldAccessor.setValue(PropertyFieldAccessorFactory.java:76)
    at org.springframework.data.neo4j.fieldaccess.DefaultEntityState.setValue(DefaultEntityState.java:113)
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.setEntityStateValue(SourceStateTransmitter.java:70)
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.access0(SourceStateTransmitter.java:40)
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.doWithPersistentProperty(SourceStateTransmitter.java:105)
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.doWithPersistentProperty(SourceStateTransmitter.java:102)
    at org.springframework.data.mapping.model.BasicPersistentEntity.doWithProperties(BasicPersistentEntity.java:307)
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.copyPropertiesTo(SourceStateTransmitter.java:102)
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityConverterImpl.write(Neo4jEntityConverterImpl.java:167)
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityPersister$CachedConverter.write(Neo4jEntityPersister.java:179)
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityPersister.persist(Neo4jEntityPersister.java:243)
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityPersister.persist(Neo4jEntityPersister.java:231)
    at org.springframework.data.neo4j.support.Neo4jTemplate.save(Neo4jTemplate.java:357)
    at org.springframework.data.neo4j.support.Neo4jTemplate.save(Neo4jTemplate.java:351)
    at org.springframework.data.neo4j.repository.AbstractGraphRepository.save(AbstractGraphRepository.java:91)
    at sun.reflect.GeneratedMethodAccessor28.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:497)
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.executeMethodOn(RepositoryFactorySupport.java:416)
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.doInvoke(RepositoryFactorySupport.java:401)
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.invoke(RepositoryFactorySupport.java:373)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$DefaultMethodInvokingMethodInterceptor.invoke(RepositoryFactorySupport.java:486)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
    at org.springframework.transaction.interceptor.TransactionInterceptor.proceedWithInvocation(TransactionInterceptor.java:99)
    at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:281)

Answer 1

您的交易量过大。

尝试创建一个只导入的服务，例如一批一千个出版物，并用@Transactional 对其进行注释（我们在该方法中使用手动事务）。然后从您的主 class 调用该方法（获取为 spring-tx-handling 注入的服务），批量为 1000 个出版物。

或者，在循环代码中使用它

tx = db.beginTx();
while () {

// do stuff

if (++counter % 1000 == 0) {
   tx.success(); tx.close();
   tx = db.beginTx();
}
}
   tx.success(); tx.close();

Spring Data Neo4j 超出了 GC 开销限制

GC overhead limit exceeded with Spring Data Neo4j

java

garbage-collection

neo4j

maven

spring-data-neo4j