Sqoop 导入 java 代码问题
Sqoop import java code issue
我是运行使用mapr提供的javajar导入sqoop，版本如下
sqoop-1.4.4-mapr-1408.jar
package com.cisco.executor;
import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.tool.ImportTool;
public class SqoopImport {
private static SqoopOptions SqoopOptions = new SqoopOptions();
private static final String connection = "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)PORT=1533))(CONNECT_DATA=(SERVICE_NAME==Dedicated)))";
private static final String username = "XXCTS_NITA_U";
private static final String password = "******";
private static final String tablename = "XXCCS_DS_SAHDR_CORE";
private static final String columns = "CONTRACT_ID,CONTRACT_NUMBER,CONTRACT_STS_CODE,CONTRACT_STATUS,SERVICE_LINE_ID,SERVICE_LINE_NAME,SERVICE_LINE_STS_CODE,SERVICE_LINE_STATUS,SERVICE_LINE_NUMBER,HDR_CUST_ACCOUNT_ID,HDR_CUST_ACCOUNT_NUMBER,HDR_PARTY_ID,HDR_CUSTOMER_NAME,LINE_CUST_ACCOUNT_ID,LINE_CUST_ACCOUNT_NUMBER,LINE_PARTY_ID,LINE_CUSTOMER_NAME,BILL_TO_SITE_USE_ID,BILL_TO_SITE_USE_NAME,BILL_TO_SITE_USE_STS,BILL_TO_CUST_ACCT_ID,BILL_TO_CUST_ACCT_NUMBER,BILL_TO_CUST_ACCT_STS,BILL_TO_CUST_ACCT_SITE_ID,BILL_TO_CUST_ACCT_SITE_STS,BILL_TO_PARTY_ID,BILL_TO_PARTY_NUMBER,BILL_TO_CUSTOMER_NAME,BILL_TO_PARTY_STS,BILL_TO_PARTY_SITE_ID,BILL_TO_PARTY_SITE_STS,BILL_TO_LOCATION_ID,BILL_TO_COUNTRY,BILL_TO_STATE_PROV,BILL_TO_CITY,BILL_TO_POSTAL_CODE,COVERAGE_TEMPLATE_ID,COVERAGE_TEMPLATE_NAME,COVERAGE_TEMPLATE_DESC,COVERAGE_BEGIN_DATE,COVERAGE_END_DATE,SERVICE_SALES_REP_USR_ID,SERVICE_SALES_REP_USR_NAME,SERVICE_LINE_INVENTORY_ITEM_ID,BILLTO_CSC_ID,BILLTO_CSC_NAME,BILLTO_GU_ID,BILLTO_GU_NAME,BILL_TO_CUSTOMER_SALES_CHANNEL,CURRENCY_CODE,CONTRACT_LIST_AMT,CONTRACT_NET_AMT,USD_CONTRACT_LIST_AMT,USD_CONTRACT_NET_AMT,ACCOUNT_TEAM,RENEWAL_SSR,RENEWAL_TEAM,SALES_REGION,ADVANCE_CONTRACT_FLAG,EARLIEST_END_DATE,SERVICE_LINE_TERMINATED_DATE,SVC_LINE_TERMINATION_REASON,CONTRACT_TERMINATED_DATE,USD_CONV_RATE,PRICE_TYPE,NET_VALUE,COTERM_DATE,MARKETING_PROGRAM,MASTER_AGREEMENT_END_DATE,DISCOUNT_PERCENT,PRORATE_FLAG,RGP_ID,CREATION_DATE,CREATED_BY,LAST_UPDATE_DATE,LAST_UPDATED_BY,LAST_UPDATED_LOGIN,REQUEST_ID,PROGRAM_UPDATE_DATE,PROGRAM_APPLICATION_ID,PROGRAM_ID,ATTRIBUTE_CATEGORY,ATTRIBUTE1,ATTRIBUTE2,ATTRIBUTE3,ATTRIBUTE4,ATTRIBUTE5,ATTRIBUTE6,ATTRIBUTE7,ATTRIBUTE8,ATTRIBUTE9,ATTRIBUTE10,ATTRIBUTE11,ATTRIBUTE12,ATTRIBUTE13,ATTRIBUTE14,ATTRIBUTE15,ATTRIBUTE16,ATTRIBUTE17,ATTRIBUTE18,ATTRIBUTE19,ATTRIBUTE20,ACCOUNT_SSR,SHELL_CONTRACT_FLAG,CONTRACT_START_DATE,CONTRACT_END_DATE";
private static final String[] cols = columns.split(",");
private static final String splitby = "CONTRACT_NUMBER";
private static final String boundaryquery = "select (select min(CONTRACT_NUMBER) from XXCCS_DS_SAHDR_CORE) as minid ,(select max(CONTRACT_NUMBER) from XXCCS_DS_SAHDR_CORE) as maxid from dual";
private static final String targetdir = "//app//dev//SmartAnalytics//sqoop_temp//XXCCS_DS_SAHDR_CORE";
private static final int mappers = 4;
private static final String nullString = "\N";
private static final String hivetable = "installbase.XXCCS_DS_SAHDR_CORE";
private static final boolean flagTrue = true;
private static final String mapcolumnshive = "CONTRACT_ID=BIGINT,SERVICE_LINE_ID=BIGINT,HDR_CUST_ACCOUNT_ID=BIGINT,HDR_PARTY_ID=BIGINT,LINE_CUST_ACCOUNT_ID=BIGINT,LINE_PARTY_ID=BIGINT,BILL_TO_SITE_USE_ID=BIGINT,BILL_TO_CUST_ACCT_ID=BIGINT,BILL_TO_CUST_ACCT_SITE_ID=BIGINT,BILL_TO_PARTY_ID=BIGINT,BILL_TO_PARTY_SITE_ID=BIGINT,BILL_TO_LOCATION_ID=BIGINT,COVERAGE_TEMPLATE_ID=BIGINT,COVERAGE_BEGIN_DATE=TIMESTAMP,COVERAGE_END_DATE=TIMESTAMP,SERVICE_LINE_INVENTORY_ITEM_ID=BIGINT,BILLTO_CSC_ID=BIGINT,BILLTO_GU_ID=BIGINT,CONTRACT_LIST_AMT=BIGINT,CONTRACT_NET_AMT=BIGINT,USD_CONTRACT_LIST_AMT=BIGINT,USD_CONTRACT_NET_AMT=BIGINT,EARLIEST_END_DATE=TIMESTAMP,SERVICE_LINE_TERMINATED_DATE=TIMESTAMP,CONTRACT_TERMINATED_DATE=TIMESTAMP,USD_CONV_RATE=BIGINT,NET_VALUE=BIGINT,MASTER_AGREEMENT_END_DATE=TIMESTAMP,DISCOUNT_PERCENT=BIGINT,PRORATE_FLAG=STRING,RGP_ID=BIGINT,CREATION_DATE=TIMESTAMP,CREATED_BY=BIGINT,LAST_UPDATE_DATE=TIMESTAMP,LAST_UPDATED_BY=BIGINT,LAST_UPDATED_LOGIN=BIGINT,REQUEST_ID=BIGINT,PROGRAM_UPDATE_DATE=TIMESTAMP,PROGRAM_APPLICATION_ID=BIGINT,PROGRAM_ID=BIGINT,ATTRIBUTE_CATEGORY=STRING,ATTRIBUTE1=STRING,ATTRIBUTE2=STRING,ATTRIBUTE3=STRING,ATTRIBUTE4=STRING,ATTRIBUTE5=STRING,ATTRIBUTE6=STRING,ATTRIBUTE7=STRING,ATTRIBUTE8=STRING,ATTRIBUTE9=STRING,ATTRIBUTE10=STRING,ATTRIBUTE11=STRING,ATTRIBUTE12=STRING,ATTRIBUTE13=STRING,ATTRIBUTE14=STRING,ATTRIBUTE15=STRING,ATTRIBUTE16=STRING,ATTRIBUTE17=STRING,ATTRIBUTE18=STRING,ATTRIBUTE19=STRING,ATTRIBUTE20=STRING,ACCOUNT_SSR=STRING,SHELL_CONTRACT_FLAG=STRING,CONTRACT_START_DATE=TIMESTAMP,CONTRACT_END_DATE=TIMESTAMP";
private static void setUp()
{
SqoopOptions.setConnectString(connection);
SqoopOptions.setUsername(username);
SqoopOptions.setPassword(password);
SqoopOptions.setTableName(tablename);
SqoopOptions.setColumns(cols);
SqoopOptions.setSplitByCol(splitby);
SqoopOptions.setBoundaryQuery(boundaryquery);
SqoopOptions.setTargetDir(targetdir);
SqoopOptions.setNumMappers(mappers);
SqoopOptions.setNullStringValue(nullString);
SqoopOptions.setNullNonStringValue(nullString);
SqoopOptions.setHiveTableName(hivetable);
SqoopOptions.setHiveImport(flagTrue);
SqoopOptions.setOverwriteHiveTable(flagTrue);
SqoopOptions.setMapColumnHive(mapcolumnshive);
}
@SuppressWarnings("deprecation")
private static int runIt() {
int res;
res = new ImportTool().run((com.cloudera.sqoop.SqoopOptions) SqoopOptions);
if (res != 0) {
throw new RuntimeException("Sqoop API Failed - return code : "+Integer.toString(res));
}
return res;
}
}
主程序：
package com.cisco.executor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
public class MainProgram {
public static void main(String[] args) {
try {
Class c = Class.forName("com.cisco.executor.SqoopImport");
Object o = c.newInstance();
Method setUp = c.getDeclaredMethod("setUp", null);
Method runIt = c.getDeclaredMethod("runIt", null);
setUp.setAccessible(true);
runIt.setAccessible(true);
setUp.invoke(o, null);
runIt.invoke(o, null);
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (SecurityException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
}
}
程序抛出如下错误
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/logging/LogFactory
at org.apache.sqoop.SqoopOptions.<clinit>(SqoopOptions.java:54)
at com.cisco.executor.SqoopImport.<clinit>(SqoopImport.java:8)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:191)
at com.cisco.executor.MainProgram.main(MainProgram.java:11)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.logging.LogFactory
at java.net.URLClassLoader.run(URLClassLoader.java:366)
at java.net.URLClassLoader.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 5 more
我尝试在 google 中搜索，但没有获得任何适当的信息来解决该问题。有人可以告诉我我的代码出了什么问题吗？
此异常意味着您的类路径中没有 apache commons-logging jar。
您需要将此 jar 添加到您的项目中。
我是运行使用mapr提供的javajar导入sqoop，版本如下
sqoop-1.4.4-mapr-1408.jar
package com.cisco.executor;
import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.tool.ImportTool;
public class SqoopImport {
private static SqoopOptions SqoopOptions = new SqoopOptions();
private static final String connection = "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)PORT=1533))(CONNECT_DATA=(SERVICE_NAME==Dedicated)))";
private static final String username = "XXCTS_NITA_U";
private static final String password = "******";
private static final String tablename = "XXCCS_DS_SAHDR_CORE";
private static final String columns = "CONTRACT_ID,CONTRACT_NUMBER,CONTRACT_STS_CODE,CONTRACT_STATUS,SERVICE_LINE_ID,SERVICE_LINE_NAME,SERVICE_LINE_STS_CODE,SERVICE_LINE_STATUS,SERVICE_LINE_NUMBER,HDR_CUST_ACCOUNT_ID,HDR_CUST_ACCOUNT_NUMBER,HDR_PARTY_ID,HDR_CUSTOMER_NAME,LINE_CUST_ACCOUNT_ID,LINE_CUST_ACCOUNT_NUMBER,LINE_PARTY_ID,LINE_CUSTOMER_NAME,BILL_TO_SITE_USE_ID,BILL_TO_SITE_USE_NAME,BILL_TO_SITE_USE_STS,BILL_TO_CUST_ACCT_ID,BILL_TO_CUST_ACCT_NUMBER,BILL_TO_CUST_ACCT_STS,BILL_TO_CUST_ACCT_SITE_ID,BILL_TO_CUST_ACCT_SITE_STS,BILL_TO_PARTY_ID,BILL_TO_PARTY_NUMBER,BILL_TO_CUSTOMER_NAME,BILL_TO_PARTY_STS,BILL_TO_PARTY_SITE_ID,BILL_TO_PARTY_SITE_STS,BILL_TO_LOCATION_ID,BILL_TO_COUNTRY,BILL_TO_STATE_PROV,BILL_TO_CITY,BILL_TO_POSTAL_CODE,COVERAGE_TEMPLATE_ID,COVERAGE_TEMPLATE_NAME,COVERAGE_TEMPLATE_DESC,COVERAGE_BEGIN_DATE,COVERAGE_END_DATE,SERVICE_SALES_REP_USR_ID,SERVICE_SALES_REP_USR_NAME,SERVICE_LINE_INVENTORY_ITEM_ID,BILLTO_CSC_ID,BILLTO_CSC_NAME,BILLTO_GU_ID,BILLTO_GU_NAME,BILL_TO_CUSTOMER_SALES_CHANNEL,CURRENCY_CODE,CONTRACT_LIST_AMT,CONTRACT_NET_AMT,USD_CONTRACT_LIST_AMT,USD_CONTRACT_NET_AMT,ACCOUNT_TEAM,RENEWAL_SSR,RENEWAL_TEAM,SALES_REGION,ADVANCE_CONTRACT_FLAG,EARLIEST_END_DATE,SERVICE_LINE_TERMINATED_DATE,SVC_LINE_TERMINATION_REASON,CONTRACT_TERMINATED_DATE,USD_CONV_RATE,PRICE_TYPE,NET_VALUE,COTERM_DATE,MARKETING_PROGRAM,MASTER_AGREEMENT_END_DATE,DISCOUNT_PERCENT,PRORATE_FLAG,RGP_ID,CREATION_DATE,CREATED_BY,LAST_UPDATE_DATE,LAST_UPDATED_BY,LAST_UPDATED_LOGIN,REQUEST_ID,PROGRAM_UPDATE_DATE,PROGRAM_APPLICATION_ID,PROGRAM_ID,ATTRIBUTE_CATEGORY,ATTRIBUTE1,ATTRIBUTE2,ATTRIBUTE3,ATTRIBUTE4,ATTRIBUTE5,ATTRIBUTE6,ATTRIBUTE7,ATTRIBUTE8,ATTRIBUTE9,ATTRIBUTE10,ATTRIBUTE11,ATTRIBUTE12,ATTRIBUTE13,ATTRIBUTE14,ATTRIBUTE15,ATTRIBUTE16,ATTRIBUTE17,ATTRIBUTE18,ATTRIBUTE19,ATTRIBUTE20,ACCOUNT_SSR,SHELL_CONTRACT_FLAG,CONTRACT_START_DATE,CONTRACT_END_DATE";
private static final String[] cols = columns.split(",");
private static final String splitby = "CONTRACT_NUMBER";
private static final String boundaryquery = "select (select min(CONTRACT_NUMBER) from XXCCS_DS_SAHDR_CORE) as minid ,(select max(CONTRACT_NUMBER) from XXCCS_DS_SAHDR_CORE) as maxid from dual";
private static final String targetdir = "//app//dev//SmartAnalytics//sqoop_temp//XXCCS_DS_SAHDR_CORE";
private static final int mappers = 4;
private static final String nullString = "\N";
private static final String hivetable = "installbase.XXCCS_DS_SAHDR_CORE";
private static final boolean flagTrue = true;
private static final String mapcolumnshive = "CONTRACT_ID=BIGINT,SERVICE_LINE_ID=BIGINT,HDR_CUST_ACCOUNT_ID=BIGINT,HDR_PARTY_ID=BIGINT,LINE_CUST_ACCOUNT_ID=BIGINT,LINE_PARTY_ID=BIGINT,BILL_TO_SITE_USE_ID=BIGINT,BILL_TO_CUST_ACCT_ID=BIGINT,BILL_TO_CUST_ACCT_SITE_ID=BIGINT,BILL_TO_PARTY_ID=BIGINT,BILL_TO_PARTY_SITE_ID=BIGINT,BILL_TO_LOCATION_ID=BIGINT,COVERAGE_TEMPLATE_ID=BIGINT,COVERAGE_BEGIN_DATE=TIMESTAMP,COVERAGE_END_DATE=TIMESTAMP,SERVICE_LINE_INVENTORY_ITEM_ID=BIGINT,BILLTO_CSC_ID=BIGINT,BILLTO_GU_ID=BIGINT,CONTRACT_LIST_AMT=BIGINT,CONTRACT_NET_AMT=BIGINT,USD_CONTRACT_LIST_AMT=BIGINT,USD_CONTRACT_NET_AMT=BIGINT,EARLIEST_END_DATE=TIMESTAMP,SERVICE_LINE_TERMINATED_DATE=TIMESTAMP,CONTRACT_TERMINATED_DATE=TIMESTAMP,USD_CONV_RATE=BIGINT,NET_VALUE=BIGINT,MASTER_AGREEMENT_END_DATE=TIMESTAMP,DISCOUNT_PERCENT=BIGINT,PRORATE_FLAG=STRING,RGP_ID=BIGINT,CREATION_DATE=TIMESTAMP,CREATED_BY=BIGINT,LAST_UPDATE_DATE=TIMESTAMP,LAST_UPDATED_BY=BIGINT,LAST_UPDATED_LOGIN=BIGINT,REQUEST_ID=BIGINT,PROGRAM_UPDATE_DATE=TIMESTAMP,PROGRAM_APPLICATION_ID=BIGINT,PROGRAM_ID=BIGINT,ATTRIBUTE_CATEGORY=STRING,ATTRIBUTE1=STRING,ATTRIBUTE2=STRING,ATTRIBUTE3=STRING,ATTRIBUTE4=STRING,ATTRIBUTE5=STRING,ATTRIBUTE6=STRING,ATTRIBUTE7=STRING,ATTRIBUTE8=STRING,ATTRIBUTE9=STRING,ATTRIBUTE10=STRING,ATTRIBUTE11=STRING,ATTRIBUTE12=STRING,ATTRIBUTE13=STRING,ATTRIBUTE14=STRING,ATTRIBUTE15=STRING,ATTRIBUTE16=STRING,ATTRIBUTE17=STRING,ATTRIBUTE18=STRING,ATTRIBUTE19=STRING,ATTRIBUTE20=STRING,ACCOUNT_SSR=STRING,SHELL_CONTRACT_FLAG=STRING,CONTRACT_START_DATE=TIMESTAMP,CONTRACT_END_DATE=TIMESTAMP";
private static void setUp()
{
SqoopOptions.setConnectString(connection);
SqoopOptions.setUsername(username);
SqoopOptions.setPassword(password);
SqoopOptions.setTableName(tablename);
SqoopOptions.setColumns(cols);
SqoopOptions.setSplitByCol(splitby);
SqoopOptions.setBoundaryQuery(boundaryquery);
SqoopOptions.setTargetDir(targetdir);
SqoopOptions.setNumMappers(mappers);
SqoopOptions.setNullStringValue(nullString);
SqoopOptions.setNullNonStringValue(nullString);
SqoopOptions.setHiveTableName(hivetable);
SqoopOptions.setHiveImport(flagTrue);
SqoopOptions.setOverwriteHiveTable(flagTrue);
SqoopOptions.setMapColumnHive(mapcolumnshive);
}
@SuppressWarnings("deprecation")
private static int runIt() {
int res;
res = new ImportTool().run((com.cloudera.sqoop.SqoopOptions) SqoopOptions);
if (res != 0) {
throw new RuntimeException("Sqoop API Failed - return code : "+Integer.toString(res));
}
return res;
}
}
主程序：
package com.cisco.executor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
public class MainProgram {
public static void main(String[] args) {
try {
Class c = Class.forName("com.cisco.executor.SqoopImport");
Object o = c.newInstance();
Method setUp = c.getDeclaredMethod("setUp", null);
Method runIt = c.getDeclaredMethod("runIt", null);
setUp.setAccessible(true);
runIt.setAccessible(true);
setUp.invoke(o, null);
runIt.invoke(o, null);
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (SecurityException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
}
}
程序抛出如下错误
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/logging/LogFactory
at org.apache.sqoop.SqoopOptions.<clinit>(SqoopOptions.java:54)
at com.cisco.executor.SqoopImport.<clinit>(SqoopImport.java:8)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:191)
at com.cisco.executor.MainProgram.main(MainProgram.java:11)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.logging.LogFactory
at java.net.URLClassLoader.run(URLClassLoader.java:366)
at java.net.URLClassLoader.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 5 more
我尝试在 google 中搜索，但没有获得任何适当的信息来解决该问题。有人可以告诉我我的代码出了什么问题吗？
此异常意味着您的类路径中没有 apache commons-logging jar。
您需要将此 jar 添加到您的项目中。