neo4j批量数据导入

package com.run.data.importData; import static org.neo4j.helpers.collection.MapUtil.map; import static org.neo4j.helpers.collection.MapUtil.stringMap; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Map; import org.neo4j.graphdb.index.BatchInserterIndex; import org.neo4j.graphdb.index.BatchInserterIndexProvider; import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider; import org.neo4j.index.impl.lucene.LuceneIndexImplementation; import org.neo4j.kernel.impl.batchinsert.BatchInserter; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; import com.run.data.Data; import com.run.data.RelType; import com.run.data.Report; /** * 批量数据导入工具 * @author shenshouer * */ public class Importer { private final String NODE_INDEX_NAME = "nodes"; private final String RELATIONSHIP_INDEX_NAME = "relationships"; /* 导入报告 */ private static Report report; /* 批量导入工具 */ private BatchInserter db; /* 批量索引建立工具 */ private BatchInserterIndexProvider indexProvider; /* 节点索引工具 */ private BatchInserterIndex nodeIndex; /* 关系索引工具 */ private BatchInserterIndex relationshipIndex; public Importer(File graphDb) { final Map<String, String> config = getConfig(); db = new BatchInserterImpl(graphDb.getAbsolutePath(), config); indexProvider = new LuceneBatchInserterIndexProvider(db); nodeIndex = indexProvider.nodeIndex(NODE_INDEX_NAME, LuceneIndexImplementation.FULLTEXT_CONFIG); relationshipIndex = indexProvider.relationshipIndex(RELATIONSHIP_INDEX_NAME, LuceneIndexImplementation.FULLTEXT_CONFIG); report = new Report(10 * 1000 * 1000, 100); } public static void main(String[] args) throws IOException { File graphDb = new File("H:/myEclipseWorkspace/neo4j-batch/target/neo4j-hello-db"); File nodesFile = new File("H:/myEclipseWorkspace/neo4j-batch/nodes.csv"); File relationshipsFile = new File("H:/myEclipseWorkspace/neo4j-batch/rels.csv"); if (!graphDb.exists()) graphDb.mkdirs(); Importer importBatch = new Importer(graphDb); try { if (nodesFile.exists()) importBatch.importNodes(nodesFile); if (relationshipsFile.exists()) importBatch.importRelationships(relationshipsFile); } finally { importBatch.finish(); } } private void importNodes(File file) throws IOException { BufferedReader bf = new BufferedReader(new FileReader(file)); final Data data = new Data(bf.readLine(), "\t", 0); String line; report.reset(); Map<String, Object> properties; long nodeId; while ((line = bf.readLine()) != null) { properties = map(data.update(line)); nodeId = db.createNode(properties); nodeIndex.add(nodeId, properties); report.dots(); } report.finishImport("Nodes"); } private void importRelationships(File file) throws IOException { BufferedReader bf = new BufferedReader(new FileReader(file)); final Data data = new Data(bf.readLine(), "\t", 3); Object[] rel = new Object[3]; final RelType type = new RelType(); String line; report.reset(); Map<String, Object> properties; long relationshipId; while ((line = bf.readLine()) != null) { properties = map(data.update(line, rel)); relationshipId = db.createRelationship(id(rel[0]), id(rel[1]), type.update(rel[2]), properties); relationshipIndex.add(relationshipId, properties); report.dots(); } report.finishImport("Relationships"); } /** * 获取配置文件 * @return */ private Map<String, String> getConfig() { if (new File("batch.properties").exists()) { return BatchInserterImpl.loadProperties("batch.properties"); } else { return stringMap( "dump_configuration", "true", "cache_type", "none", "neostore.propertystore.db.index.keys.mapped_memory", "5M", "neostore.propertystore.db.index.mapped_memory", "5M", "neostore.nodestore.db.mapped_memory", "50M", "neostore.relationshipstore.db.mapped_memory", "250M", "neostore.propertystore.db.mapped_memory", "200M", "neostore.propertystore.db.strings.mapped_memory", "100M"); } } private long id(Object id) { return Long.parseLong(id.toString()); } private void finish() { db.shutdown(); report.finish(); } }