通过通用标签有效地合并 java 中的 2 个大型 csv 文件
Efficiently merge 2 large csv files in java by common labels
我需要通过常见的行或列标签合并 2 个大型 csv 文件（每个文件大约有 4000 万个数据元素 ~500mb），这些标签可以由用户指定。例如，如果 dataset1.csv 包含：
patient_id x1 x2 x3
pi1 1 2 3
pi3 4 5 6
和dataset2.csv包含：
patient_id y1 y2 y3
pi0 0 0 0
pi1 11 12 13
pi2 99 98 97
pi3 14 15 16
用户可以指定按行标签（患者 ID）合并这两个文件，结果 output.csv 将是：
patient_id x1 x2 x3 y1 y2 y3
pi1 1 2 3 11 12 13
pi3 4 5 6 14 15 16
因为我们只合并了两个输入文件共有（交集）的患者 ID 信息。我解决这个问题的策略是创建一个 HashMap，其中要合并的行或列标签（在本例中是行标签，即患者 ID）是键，患者 ID 的数据存储为 ArrayList 作为价值。我为每个输入数据文件创建一个 HashMap，然后根据相似的键合并这些值。我将数据表示为 ArrayList> 类型的二维 ArrayList，因此合并后的数据也具有此类型。然后我简单地遍历合并的 ArrayList> 对象，我称之为数据类型对象，并将它打印到文件中。代码如下：
下面是依赖于下面的数据 class 文件的 DataMerge class。
import java.util.HashMap;
import java.util.ArrayList;
public class DataMerge {
/**Merges two Data objects by a similar label. For example, if two data sets represent
* different data for the same set of patients, which are represented by their unique patient
* ID, mergeData will return a data set containing only those patient IDs that are common to both
* data sets along with the data represented in both data sets. labelInRow1 and labelInRow2 separately
* indicate whether the common labels are in separate rows(true) of d1 and d2, respectively, or separate columns otherwise.*/
public static Data mergeData(Data d1, Data d2, boolean labelInRow1,
boolean labelInRow2){
ArrayList<ArrayList<String>> mergedData = new ArrayList<ArrayList<String>>();
HashMap<String,ArrayList<String>> d1Map = d1.mapFeatureToData(labelInRow1);
HashMap<String,ArrayList<String>> d2Map = d2.mapFeatureToData(labelInRow2);
ArrayList<String> d1Features;
ArrayList<String> d2Features;
if (labelInRow1){
d1Features = d1.getColumnLabels();
} else {
d1Features = d1.getRowLabels();
}
if (labelInRow2){
d2Features = d2.getColumnLabels();
} else {
d2Features = d2.getRowLabels();
}
d1Features.trimToSize();
d2Features.trimToSize();
ArrayList<String> mergedFeatures = new ArrayList<String>();
if ((d1.getLabelLabel() != "") && (d1.getLabelLabel() == "")) {
mergedFeatures.add(d1.getLabelLabel());
}
else if ((d1.getLabelLabel() == "") && (d1.getLabelLabel() != "")) {
mergedFeatures.add(d2.getLabelLabel());
} else {
mergedFeatures.add(d1.getLabelLabel());
}
mergedFeatures.addAll(d1Features);
mergedFeatures.addAll(d2Features);
mergedFeatures.trimToSize();
mergedData.add(mergedFeatures);
for (String key : d1Map.keySet()){
ArrayList<String> curRow = new ArrayList<String>();
if (d2Map.containsKey(key)){
curRow.add(key);
curRow.addAll(d1Map.get(key));
curRow.addAll(d2Map.get(key));
curRow.trimToSize();
mergedData.add(curRow);
}
}
mergedData.trimToSize();
Data result = new Data(mergedData, true);
return result;
}
}
下面是数据类型对象及其关联的 HashMap 生成函数以及一些行和列标签提取方法。
import java.util.*;
import java.io.*;
/**Represents an unlabeled or labeled data set as a series of nested ArrayLists, where each nested
* ArrayList represents a line of the input data.*/
public class Data {
private ArrayList<String> colLabels = new ArrayList<String>(); //row labels
private ArrayList<String> rowLabels = new ArrayList<String>(); //column labels
private String labelLabel;
private ArrayList<ArrayList<String>> unlabeledData; //data without row and column labels
/**Returns an ArrayList of ArrayLists, where each nested ArrayList represents a line
*of the input file.*/
@SuppressWarnings("resource")
private static ArrayList<ArrayList<String>> readFile(String filePath, String fileSep){
ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
try{
BufferedReader input = new BufferedReader(new FileReader (filePath));
String line = input.readLine();
while (line != null){
String[] splitLine = line.split(fileSep);
result.add(new ArrayList<String>(Arrays.asList(splitLine)));
line = input.readLine();
}
}
catch (Exception e){
System.err.println(e);
}
result.trimToSize();;
return result;
}
/**Returns an ArrayList of ArrayLists, where each nested ArrayList represents a line of the input
* data but WITHOUT any row or column labels*/
private ArrayList<ArrayList<String>> extractLabelsAndData(String filePath, String fileSep){
ArrayList<ArrayList<String>> tempData = new ArrayList<ArrayList<String>>();
tempData.addAll(readFile(filePath, fileSep));
tempData.trimToSize();
this.colLabels.addAll(tempData.remove(0));
this.labelLabel = this.colLabels.remove(0);
this.colLabels.trimToSize();
for (ArrayList<String> line : tempData){
this.rowLabels.add(line.remove(0));
}
this.rowLabels.trimToSize();
return tempData;
}
/**Returns an ArrayList of ArrayLists, where each nested ArrayList represents a line of the input
* data but WITHOUT any row or column labels. Does mutate the original data*/
private ArrayList<ArrayList<String>> extractLabelsAndData (ArrayList<ArrayList<String>> data){
ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
for (ArrayList<String> line : data){
ArrayList<String> temp = new ArrayList<String>();
for (String element : line){
temp.add(element);
}
temp.trimToSize();
result.add(temp);
}
this.colLabels.addAll(result.remove(0));
this.labelLabel = this.colLabels.remove(0);
this.colLabels.trimToSize();
for (ArrayList<String> line : result){
this.rowLabels.add(line.remove(0));
}
this.rowLabels.trimToSize();
result.trimToSize();
return result;
}
/**Returns the labelLabel for the data*/
public String getLabelLabel(){
return this.labelLabel;
}
/**Returns an ArrayList of the labels while maintaining the order
* in which they appear in the data. Row indicates that the desired
* features are all in the same row. Assumed that the labels are in the
* first row of the data. */
public ArrayList<String> getColumnLabels(){
return this.colLabels;
}
/**Returns an ArrayList of the labels while maintaining the order
* in which they appear in the data. Column indicates that the desired
* features are all in the same column. Assumed that the labels are in the
* first column of the data.*/
public ArrayList<String> getRowLabels(){
return this.rowLabels;
}
/**Creates a HashMap where a list of feature labels are mapped to the entire data. For example,
* if a data set contains patient IDs and test results, this function can be used to create
* a HashMap where the keys are the patient IDs and the values are an ArrayList of the test
* results. The boolean input isRow, which, when true, designates that the
* desired keys are listed in the rows or false if they are in the columns.*/
public HashMap<String, ArrayList<String>> mapFeatureToData(boolean isRow){
HashMap<String, ArrayList<String>> featureMap = new HashMap<String,ArrayList<String>>();
if (!isRow){
for (ArrayList<String> line : this.unlabeledData){
for (int i = 0; i < this.colLabels.size(); i++){
if (featureMap.containsKey(this.colLabels.get(i))){
featureMap.get(this.colLabels.get(i)).add(line.get(i));
} else{
ArrayList<String> firstValue = new ArrayList<String>();
firstValue.add(line.get(i));
featureMap.put(this.colLabels.get(i), firstValue);
}
}
}
} else {
for (int i = 0; i < this.rowLabels.size(); i++){
if (!featureMap.containsKey(this.rowLabels.get(i))){
featureMap.put(this.rowLabels.get(i), this.unlabeledData.get(i));
} else {
featureMap.get(this.rowLabels.get(i)).addAll(this.unlabeledData.get(i));
}
}
}
return featureMap;
}
/**Writes the data to a file in the specified outputPath. sep indicates the data delimiter.
* labeledOutput indicates whether or not the user wants the data written to a file to be
* labeled or unlabeled. If the data was unlabeled to begin with, then labeledOutput
* should not be set to true. */
public void writeDataToFile(String outputPath, String sep){
try {
PrintStream writer = new PrintStream(new BufferedOutputStream (new FileOutputStream (outputPath, true)));
String sol = this.labelLabel + sep;
for (int n = 0; n < this.colLabels.size(); n++){
if (n == this.colLabels.size()-1){
sol += this.colLabels.get(n) + "\n";
} else {
sol += this.colLabels.get(n) + sep;
}
}
for (int i = 0; i < this.unlabeledData.size(); i++){
ArrayList<String> line = this.unlabeledData.get(i);
sol += this.rowLabels.get(i) + sep;
for (int j = 0; j < line.size(); j++){
if (j == line.size()-1){
sol += line.get(j);
} else {
sol += line.get(j) + sep;
}
}
sol += "\n";
}
sol = sol.trim();
writer.print(sol);
writer.close();
} catch (Exception e){
System.err.println(e);
}
}
/**Constructor for Data object. filePath specifies the input file directory,
* fileSep indicates the file separator used in the input file, and hasLabels
* designates whether the input data has row and column labels. Note that if
* hasLabels is set to true, it is assumed that there are BOTH row and column labels*/
public Data(String filePath, String fileSep, boolean hasLabels){
if (hasLabels){
this.unlabeledData = extractLabelsAndData(filePath, fileSep);
this.unlabeledData.trimToSize();
} else {
this.unlabeledData = readFile(filePath, fileSep);
this.unlabeledData.trimToSize();
}
}
/**Constructor for Data object that accepts nested ArrayLists as inputs*/
public Data (ArrayList<ArrayList<String>> data, boolean hasLabels){
if (hasLabels){
this.unlabeledData = extractLabelsAndData(data);
this.unlabeledData.trimToSize();
} else {
this.unlabeledData = data;
this.unlabeledData.trimToSize();
}
}
}
该程序适用于小型数据集，但已经 5 天多了，合并仍未完成。我正在寻找更有效的时间和内存解决方案。有人建议使用字节数组而不是字符串，这可能会使它 运行 更快。有人有什么建议吗？
编辑：我在我的代码中做了一些挖掘，发现读取输入文件并合并它们几乎不需要时间（比如 20 秒）。写文件是需要5+天的部分
您正在将所有数百万行数据的所有数据字段连接成一个巨大的字符串，然后写入该单个字符串。当您分配和重新分配非常大的字符串时，这是由于内存抖动导致的缓慢死亡，一遍又一遍地复制它们 每个字段和分隔符 您要添加到字符串中。在第 3 天或第 4 天左右，每个字符串都是……数百万个字符长？ ......而你可怜的垃圾收集者正在大汗淋漓地向你发泄。
不要那样做。
分别构建输出文件的每一行并写入。然后构建下一行。
此外，使用 StringBuilder class 来构建线条，尽管您会在上一步中获得这样的改进，但您甚至可能不会为此烦恼。虽然这是这样做的方式，但您应该学习如何做。
我需要通过常见的行或列标签合并 2 个大型 csv 文件（每个文件大约有 4000 万个数据元素 ~500mb），这些标签可以由用户指定。例如，如果 dataset1.csv 包含：
patient_id x1 x2 x3
pi1 1 2 3
pi3 4 5 6
和dataset2.csv包含：
patient_id y1 y2 y3
pi0 0 0 0
pi1 11 12 13
pi2 99 98 97
pi3 14 15 16
用户可以指定按行标签（患者 ID）合并这两个文件，结果 output.csv 将是：
patient_id x1 x2 x3 y1 y2 y3
pi1 1 2 3 11 12 13
pi3 4 5 6 14 15 16
因为我们只合并了两个输入文件共有（交集）的患者 ID 信息。我解决这个问题的策略是创建一个 HashMap，其中要合并的行或列标签（在本例中是行标签，即患者 ID）是键，患者 ID 的数据存储为 ArrayList 作为价值。我为每个输入数据文件创建一个 HashMap，然后根据相似的键合并这些值。我将数据表示为 ArrayList> 类型的二维 ArrayList，因此合并后的数据也具有此类型。然后我简单地遍历合并的 ArrayList> 对象，我称之为数据类型对象，并将它打印到文件中。代码如下：
下面是依赖于下面的数据 class 文件的 DataMerge class。
import java.util.HashMap;
import java.util.ArrayList;
public class DataMerge {
/**Merges two Data objects by a similar label. For example, if two data sets represent
* different data for the same set of patients, which are represented by their unique patient
* ID, mergeData will return a data set containing only those patient IDs that are common to both
* data sets along with the data represented in both data sets. labelInRow1 and labelInRow2 separately
* indicate whether the common labels are in separate rows(true) of d1 and d2, respectively, or separate columns otherwise.*/
public static Data mergeData(Data d1, Data d2, boolean labelInRow1,
boolean labelInRow2){
ArrayList<ArrayList<String>> mergedData = new ArrayList<ArrayList<String>>();
HashMap<String,ArrayList<String>> d1Map = d1.mapFeatureToData(labelInRow1);
HashMap<String,ArrayList<String>> d2Map = d2.mapFeatureToData(labelInRow2);
ArrayList<String> d1Features;
ArrayList<String> d2Features;
if (labelInRow1){
d1Features = d1.getColumnLabels();
} else {
d1Features = d1.getRowLabels();
}
if (labelInRow2){
d2Features = d2.getColumnLabels();
} else {
d2Features = d2.getRowLabels();
}
d1Features.trimToSize();
d2Features.trimToSize();
ArrayList<String> mergedFeatures = new ArrayList<String>();
if ((d1.getLabelLabel() != "") && (d1.getLabelLabel() == "")) {
mergedFeatures.add(d1.getLabelLabel());
}
else if ((d1.getLabelLabel() == "") && (d1.getLabelLabel() != "")) {
mergedFeatures.add(d2.getLabelLabel());
} else {
mergedFeatures.add(d1.getLabelLabel());
}
mergedFeatures.addAll(d1Features);
mergedFeatures.addAll(d2Features);
mergedFeatures.trimToSize();
mergedData.add(mergedFeatures);
for (String key : d1Map.keySet()){
ArrayList<String> curRow = new ArrayList<String>();
if (d2Map.containsKey(key)){
curRow.add(key);
curRow.addAll(d1Map.get(key));
curRow.addAll(d2Map.get(key));
curRow.trimToSize();
mergedData.add(curRow);
}
}
mergedData.trimToSize();
Data result = new Data(mergedData, true);
return result;
}
}
下面是数据类型对象及其关联的 HashMap 生成函数以及一些行和列标签提取方法。
import java.util.*;
import java.io.*;
/**Represents an unlabeled or labeled data set as a series of nested ArrayLists, where each nested
* ArrayList represents a line of the input data.*/
public class Data {
private ArrayList<String> colLabels = new ArrayList<String>(); //row labels
private ArrayList<String> rowLabels = new ArrayList<String>(); //column labels
private String labelLabel;
private ArrayList<ArrayList<String>> unlabeledData; //data without row and column labels
/**Returns an ArrayList of ArrayLists, where each nested ArrayList represents a line
*of the input file.*/
@SuppressWarnings("resource")
private static ArrayList<ArrayList<String>> readFile(String filePath, String fileSep){
ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
try{
BufferedReader input = new BufferedReader(new FileReader (filePath));
String line = input.readLine();
while (line != null){
String[] splitLine = line.split(fileSep);
result.add(new ArrayList<String>(Arrays.asList(splitLine)));
line = input.readLine();
}
}
catch (Exception e){
System.err.println(e);
}
result.trimToSize();;
return result;
}
/**Returns an ArrayList of ArrayLists, where each nested ArrayList represents a line of the input
* data but WITHOUT any row or column labels*/
private ArrayList<ArrayList<String>> extractLabelsAndData(String filePath, String fileSep){
ArrayList<ArrayList<String>> tempData = new ArrayList<ArrayList<String>>();
tempData.addAll(readFile(filePath, fileSep));
tempData.trimToSize();
this.colLabels.addAll(tempData.remove(0));
this.labelLabel = this.colLabels.remove(0);
this.colLabels.trimToSize();
for (ArrayList<String> line : tempData){
this.rowLabels.add(line.remove(0));
}
this.rowLabels.trimToSize();
return tempData;
}
/**Returns an ArrayList of ArrayLists, where each nested ArrayList represents a line of the input
* data but WITHOUT any row or column labels. Does mutate the original data*/
private ArrayList<ArrayList<String>> extractLabelsAndData (ArrayList<ArrayList<String>> data){
ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
for (ArrayList<String> line : data){
ArrayList<String> temp = new ArrayList<String>();
for (String element : line){
temp.add(element);
}
temp.trimToSize();
result.add(temp);
}
this.colLabels.addAll(result.remove(0));
this.labelLabel = this.colLabels.remove(0);
this.colLabels.trimToSize();
for (ArrayList<String> line : result){
this.rowLabels.add(line.remove(0));
}
this.rowLabels.trimToSize();
result.trimToSize();
return result;
}
/**Returns the labelLabel for the data*/
public String getLabelLabel(){
return this.labelLabel;
}
/**Returns an ArrayList of the labels while maintaining the order
* in which they appear in the data. Row indicates that the desired
* features are all in the same row. Assumed that the labels are in the
* first row of the data. */
public ArrayList<String> getColumnLabels(){
return this.colLabels;
}
/**Returns an ArrayList of the labels while maintaining the order
* in which they appear in the data. Column indicates that the desired
* features are all in the same column. Assumed that the labels are in the
* first column of the data.*/
public ArrayList<String> getRowLabels(){
return this.rowLabels;
}
/**Creates a HashMap where a list of feature labels are mapped to the entire data. For example,
* if a data set contains patient IDs and test results, this function can be used to create
* a HashMap where the keys are the patient IDs and the values are an ArrayList of the test
* results. The boolean input isRow, which, when true, designates that the
* desired keys are listed in the rows or false if they are in the columns.*/
public HashMap<String, ArrayList<String>> mapFeatureToData(boolean isRow){
HashMap<String, ArrayList<String>> featureMap = new HashMap<String,ArrayList<String>>();
if (!isRow){
for (ArrayList<String> line : this.unlabeledData){
for (int i = 0; i < this.colLabels.size(); i++){
if (featureMap.containsKey(this.colLabels.get(i))){
featureMap.get(this.colLabels.get(i)).add(line.get(i));
} else{
ArrayList<String> firstValue = new ArrayList<String>();
firstValue.add(line.get(i));
featureMap.put(this.colLabels.get(i), firstValue);
}
}
}
} else {
for (int i = 0; i < this.rowLabels.size(); i++){
if (!featureMap.containsKey(this.rowLabels.get(i))){
featureMap.put(this.rowLabels.get(i), this.unlabeledData.get(i));
} else {
featureMap.get(this.rowLabels.get(i)).addAll(this.unlabeledData.get(i));
}
}
}
return featureMap;
}
/**Writes the data to a file in the specified outputPath. sep indicates the data delimiter.
* labeledOutput indicates whether or not the user wants the data written to a file to be
* labeled or unlabeled. If the data was unlabeled to begin with, then labeledOutput
* should not be set to true. */
public void writeDataToFile(String outputPath, String sep){
try {
PrintStream writer = new PrintStream(new BufferedOutputStream (new FileOutputStream (outputPath, true)));
String sol = this.labelLabel + sep;
for (int n = 0; n < this.colLabels.size(); n++){
if (n == this.colLabels.size()-1){
sol += this.colLabels.get(n) + "\n";
} else {
sol += this.colLabels.get(n) + sep;
}
}
for (int i = 0; i < this.unlabeledData.size(); i++){
ArrayList<String> line = this.unlabeledData.get(i);
sol += this.rowLabels.get(i) + sep;
for (int j = 0; j < line.size(); j++){
if (j == line.size()-1){
sol += line.get(j);
} else {
sol += line.get(j) + sep;
}
}
sol += "\n";
}
sol = sol.trim();
writer.print(sol);
writer.close();
} catch (Exception e){
System.err.println(e);
}
}
/**Constructor for Data object. filePath specifies the input file directory,
* fileSep indicates the file separator used in the input file, and hasLabels
* designates whether the input data has row and column labels. Note that if
* hasLabels is set to true, it is assumed that there are BOTH row and column labels*/
public Data(String filePath, String fileSep, boolean hasLabels){
if (hasLabels){
this.unlabeledData = extractLabelsAndData(filePath, fileSep);
this.unlabeledData.trimToSize();
} else {
this.unlabeledData = readFile(filePath, fileSep);
this.unlabeledData.trimToSize();
}
}
/**Constructor for Data object that accepts nested ArrayLists as inputs*/
public Data (ArrayList<ArrayList<String>> data, boolean hasLabels){
if (hasLabels){
this.unlabeledData = extractLabelsAndData(data);
this.unlabeledData.trimToSize();
} else {
this.unlabeledData = data;
this.unlabeledData.trimToSize();
}
}
}
该程序适用于小型数据集，但已经 5 天多了，合并仍未完成。我正在寻找更有效的时间和内存解决方案。有人建议使用字节数组而不是字符串，这可能会使它 运行 更快。有人有什么建议吗？
编辑：我在我的代码中做了一些挖掘，发现读取输入文件并合并它们几乎不需要时间（比如 20 秒）。写文件是需要5+天的部分
您正在将所有数百万行数据的所有数据字段连接成一个巨大的字符串，然后写入该单个字符串。当您分配和重新分配非常大的字符串时，这是由于内存抖动导致的缓慢死亡，一遍又一遍地复制它们 每个字段和分隔符 您要添加到字符串中。在第 3 天或第 4 天左右，每个字符串都是……数百万个字符长？ ......而你可怜的垃圾收集者正在大汗淋漓地向你发泄。
不要那样做。
分别构建输出文件的每一行并写入。然后构建下一行。
此外，使用 StringBuilder class 来构建线条，尽管您会在上一步中获得这样的改进，但您甚至可能不会为此烦恼。虽然这是这样做的方式，但您应该学习如何做。