无法连接 Spark-Cloudant
Not able to connect Spark-Cloudant
我正在尝试使用 Java 代码从 Cloudant 获取数据并出现错误,
我尝试使用以下 Spark 和 cloudant-spark 版本,
Spark 2.0.0,
Spark 2.0.1,
Spark 2.0.2
所有版本都出现与下面发布的错误相同的错误。
如果我添加 scala 依赖项来解决此错误,则此错误与 Spark 库冲突。
下面是我的java代码,
package spark.cloudant.connecter;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SQLContext;
import com.cloudant.spark.*;
public class cloudantconnecter {
public static void main(String[] args) throws Exception {
try {
SparkConf sparkConf = new SparkConf().setAppName("spark cloudant connecter").setMaster("local[*]");
sparkConf.set("spark.streaming.concurrentJobs", "30");
JavaSparkContext sc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(sc);
System.out.print("initialization successfully");
Dataset<org.apache.spark.sql.Row> st = sqlContext.read().format("com.cloudant.spark")
.option("cloudant.host", "HOSTNAME").option("cloudant.username", "USERNAME")
.option("cloudant.password", "PASSWORD").load("DATABASENAME");
st.printSchema();
} catch (
Exception e) {
e.printStackTrace();
}
}
}
Maven 依赖项
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.10</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>cloudant-labs</groupId>
<artifactId>spark-cloudant</artifactId>
<version>2.0.0-s_2.11</version>
</dependency>
</dependencies>
正在获取错误详细信息,
Exception in thread "main" java.lang.NoSuchMethodError: scala/Predef$.ArrowAssoc(Ljava/lang/Object;)Ljava/lang/Object; (loaded from file:/C:/Users/Administrator/.m2/repository/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar by sun.misc.Launcher$AppClassLoader@9f916f97) called from class scalaj.http.HttpConstants$ (loaded from file:/C:/Users/Administrator/.m2/repository/org/scalaj/scalaj-http_2.11/2.3.0/scalaj-http_2.11-2.3.0.jar by sun.misc.Launcher$AppClassLoader@9f916f97).
at scalaj.http.HttpConstants$.liftedTree1(Http.scala:637)
at scalaj.http.HttpConstants$.<init>(Http.scala:636)
at scalaj.http.HttpConstants$.<clinit>(Http.scala)
at scalaj.http.BaseHttp$.$lessinit$greater$default(Http.scala:754)
at scalaj.http.Http$.<init>(Http.scala:738)
at scalaj.http.Http$.<clinit>(Http.scala)
at com.cloudant.spark.common.JsonStoreDataAccess.getQueryResult(JsonStoreDataAccess.scala:152)
at com.cloudant.spark.common.JsonStoreDataAccess.getTotalRows(JsonStoreDataAccess.scala:99)
at com.cloudant.spark.common.JsonStoreRDD.totalRows$lzycompute(JsonStoreRDD.scala:56)
at com.cloudant.spark.common.JsonStoreRDD.totalRows(JsonStoreRDD.scala:55)
at com.cloudant.spark.common.JsonStoreRDD.totalPartition$lzycompute(JsonStoreRDD.scala:59)
at com.cloudant.spark.common.JsonStoreRDD.totalPartition(JsonStoreRDD.scala:58)
at com.cloudant.spark.common.JsonStoreRDD.getPartitions(JsonStoreRDD.scala:81)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:248)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:246)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:248)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:246)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1934)
at org.apache.spark.rdd.RDD$$anonfun$fold.apply(RDD.scala:1046)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
at org.apache.spark.rdd.RDD.fold(RDD.scala:1040)
at org.apache.spark.sql.execution.datasources.json.InferSchema$.infer(InferSchema.scala:68)
at org.apache.spark.sql.DataFrameReader$$anonfun.apply(DataFrameReader.scala:317)
at org.apache.spark.sql.DataFrameReader$$anonfun.apply(DataFrameReader.scala:317)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:316)
at com.cloudant.spark.DefaultSource.create(DefaultSource.scala:127)
at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:105)
at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:100)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:315)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:149)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:132)
at spark.cloudant.connecter.cloudantconnecter.main(cloudantconnecter.java:24)
显示错误是因为提到的库使用 scala 2.10,提到的包 spark cloudant 库使用 2.11
所以请将库 spark-core_2.10 更改为 spark-core_2.11
所以现在依赖关系是,
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.11</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>cloudant-labs</groupId>
<artifactId>spark-cloudant</artifactId>
<version>2.0.0-s_2.11</version>
</dependency>
我正在尝试使用 Java 代码从 Cloudant 获取数据并出现错误,
我尝试使用以下 Spark 和 cloudant-spark 版本,
Spark 2.0.0,
Spark 2.0.1,
Spark 2.0.2
所有版本都出现与下面发布的错误相同的错误。
如果我添加 scala 依赖项来解决此错误,则此错误与 Spark 库冲突。
下面是我的java代码,
package spark.cloudant.connecter;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SQLContext;
import com.cloudant.spark.*;
public class cloudantconnecter {
public static void main(String[] args) throws Exception {
try {
SparkConf sparkConf = new SparkConf().setAppName("spark cloudant connecter").setMaster("local[*]");
sparkConf.set("spark.streaming.concurrentJobs", "30");
JavaSparkContext sc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(sc);
System.out.print("initialization successfully");
Dataset<org.apache.spark.sql.Row> st = sqlContext.read().format("com.cloudant.spark")
.option("cloudant.host", "HOSTNAME").option("cloudant.username", "USERNAME")
.option("cloudant.password", "PASSWORD").load("DATABASENAME");
st.printSchema();
} catch (
Exception e) {
e.printStackTrace();
}
}
}
Maven 依赖项
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.10</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>cloudant-labs</groupId>
<artifactId>spark-cloudant</artifactId>
<version>2.0.0-s_2.11</version>
</dependency>
</dependencies>
正在获取错误详细信息,
Exception in thread "main" java.lang.NoSuchMethodError: scala/Predef$.ArrowAssoc(Ljava/lang/Object;)Ljava/lang/Object; (loaded from file:/C:/Users/Administrator/.m2/repository/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar by sun.misc.Launcher$AppClassLoader@9f916f97) called from class scalaj.http.HttpConstants$ (loaded from file:/C:/Users/Administrator/.m2/repository/org/scalaj/scalaj-http_2.11/2.3.0/scalaj-http_2.11-2.3.0.jar by sun.misc.Launcher$AppClassLoader@9f916f97).
at scalaj.http.HttpConstants$.liftedTree1(Http.scala:637)
at scalaj.http.HttpConstants$.<init>(Http.scala:636)
at scalaj.http.HttpConstants$.<clinit>(Http.scala)
at scalaj.http.BaseHttp$.$lessinit$greater$default(Http.scala:754)
at scalaj.http.Http$.<init>(Http.scala:738)
at scalaj.http.Http$.<clinit>(Http.scala)
at com.cloudant.spark.common.JsonStoreDataAccess.getQueryResult(JsonStoreDataAccess.scala:152)
at com.cloudant.spark.common.JsonStoreDataAccess.getTotalRows(JsonStoreDataAccess.scala:99)
at com.cloudant.spark.common.JsonStoreRDD.totalRows$lzycompute(JsonStoreRDD.scala:56)
at com.cloudant.spark.common.JsonStoreRDD.totalRows(JsonStoreRDD.scala:55)
at com.cloudant.spark.common.JsonStoreRDD.totalPartition$lzycompute(JsonStoreRDD.scala:59)
at com.cloudant.spark.common.JsonStoreRDD.totalPartition(JsonStoreRDD.scala:58)
at com.cloudant.spark.common.JsonStoreRDD.getPartitions(JsonStoreRDD.scala:81)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:248)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:246)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:248)
at org.apache.spark.rdd.RDD$$anonfun$partitions.apply(RDD.scala:246)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1934)
at org.apache.spark.rdd.RDD$$anonfun$fold.apply(RDD.scala:1046)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
at org.apache.spark.rdd.RDD.fold(RDD.scala:1040)
at org.apache.spark.sql.execution.datasources.json.InferSchema$.infer(InferSchema.scala:68)
at org.apache.spark.sql.DataFrameReader$$anonfun.apply(DataFrameReader.scala:317)
at org.apache.spark.sql.DataFrameReader$$anonfun.apply(DataFrameReader.scala:317)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:316)
at com.cloudant.spark.DefaultSource.create(DefaultSource.scala:127)
at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:105)
at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:100)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:315)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:149)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:132)
at spark.cloudant.connecter.cloudantconnecter.main(cloudantconnecter.java:24)
显示错误是因为提到的库使用 scala 2.10,提到的包 spark cloudant 库使用 2.11
所以请将库 spark-core_2.10 更改为 spark-core_2.11
所以现在依赖关系是,
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.11</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>cloudant-labs</groupId>
<artifactId>spark-cloudant</artifactId>
<version>2.0.0-s_2.11</version>
</dependency>