Apache ignite and spark itegration error java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.createDataFrame

Question

我们正在尝试集成 apache ignite 和 apache spark，但出现以下错误。

Maven 配置

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.me</groupId>
    <artifactId>igniteclient1</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>

    <name>igniteclient1</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>2.1.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.10 -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.10</artifactId>
            <version>2.1.0</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-core -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-core</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-spring -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-spring</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-indexing -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-indexing</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-rest-http -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-rest-http</artifactId>
            <version>1.8.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.ignite/ignite-spark -->
        <dependency>
            <groupId>org.apache.ignite</groupId>
            <artifactId>ignite-spark</artifactId>
            <version>1.8.0</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.0.0</version>
                <configuration>
                    <filters>
                        <filter>
                            <artifact>*:*</artifact>
                            <excludes>
                                <exclude>META-INF/*.SF</exclude>
                                <exclude>META-INF/*.DSA</exclude>
                                <exclude>META-INF/*.RSA</exclude>
                            </excludes>
                        </filter>
                    </filters>
                    <!-- Additional configuration. -->
                </configuration>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

代码：

package com.me.igniteclient1;

import org.apache.spark.api.java.*;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;

import com.me.igniteclient1.ratio.Ratio;

import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.spark.JavaIgniteContext;
import org.apache.ignite.spark.JavaIgniteRDD;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.SQLContext;


/**
 *
 */
public class App 
{
    public static void main(String[] args) {

        SparkConf conf = new SparkConf().setAppName("Stockrank styles");
        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaIgniteContext igniteContext = new JavaIgniteContext(sc, new IgniteConfigProvider());
        JavaIgniteRDD<String, Ratio> javaRdd = igniteContext.fromCache("RatioCache");

        DataFrame df = javaRdd.sql("select ric, ratio1, ratio2, ratio3, ratio4 from Ratio where date = ? and ratios__Exchange = ? and ratio5 > ?", "2017-01-25", "LSE", 10);

        df.printSchema();
        df.show();

        sc.stop();
      }

    /**
     * Ignite configiration provider.
     */
    static class IgniteConfigProvider implements IgniteOutClosure<IgniteConfiguration> {

        public IgniteConfiguration apply() {

            Ignition.setClientMode(true);

            // Start Ignite in client mode.
            Ignite ignite = Ignition.start();
            return ignite.configuration();
        }

    }

}

错误：

 Exception in thread "main" java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.createDataFrame(Lorg/apache/spark/rdd/RDD;Lorg/apache/spark/sql/types/StructType;)Lorg/apache/spark/sql/DataFrame;
        at org.apache.ignite.spark.IgniteRDD.sql(IgniteRDD.scala:152)
        at org.apache.ignite.spark.JavaIgniteRDD.sql(JavaIgniteRDD.scala:79)
        at org.apache.ignite.spark.JavaIgniteRDD.sql(JavaIgniteRDD.scala:79)
        at com.me.igniteclient1.App.main(App.java:38)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain(SparkSubmit.scala:187)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
    17/02/10 17:55:10 INFO GridUpdateNotifier: Your version is up to date.

这些测试在 ignite spark 模块中通过了吗？

https://github.com/apache/ignite/blob/master/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java

https://github.com/apache/ignite/blob/master/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java

我尝试了不同的 spark 版本，1.6、1.5、1.4，但没有任何效果。我收到同样的错误。

我们正在评估使用 ignite（作为内存数据结构）和 spark（使用 mlib 进行分析）的可能性。但是集成模块好像不是很活跃。未来的计划是什么？它会被弃用吗？

更新：Ignite 1.9 发布https://blogs.apache.org/ignite/entry/apache-ignite-1-9-released

Spark

Ignite’s spark integration was upgraded to the latest Spark version. Presently, you can leverage from Ignite Shared RDDs in applications using latest Spark version.

Answer 1

根据 POM，您使用的是 Spark 2.1.0。 Ignite基于1.5.2，目前不支持2.x.

还为 Scala 2.11 创建了 ignite-spark 模块。如果您有 2.10，请改用 ignite-spark_2.10。

总的来说，这绝对是一个类路径问题。确保所有版本一致，它应该可以正常工作。

Apache ignite and spark itegration error java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.createDataFrame

Apache ignite and spark itegration error java.lang.NoSuchMethodError: org.apache.spark.sql.SQLContext.createDataFrame

java

gridgain

apache-spark

ignite