Apache Storm 应用程序在 运行 失败
Apache Storm application fails at run
我在运行时遇到问题，当我的应用程序在编译时找不到在 Maven 中列为依赖项的 类。应用程序编译没有问题。
申请如下
package com.ender.storm;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
public class HelloStorm {
public static void main(String[] args) throws Exception{
Config config = new Config();
config.put("inputFile", args[0]);
config.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
config.put(Config.TOPOLOGY_DEBUG, false);
config.setDebug(false);
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("line-reader-spout", new LineReaderSpout());
builder.setBolt("word-spitter", new WordSpitterBolt()).shuffleGrouping("line-reader-spout");
builder.setBolt("word-counter", new WordCounterBolt()).shuffleGrouping("word-spitter");
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("HelloStorm", config, builder.createTopology());
Thread.sleep(10000);
cluster.shutdown();
}
}
LineReaderSpout 如下所示
包 com.ender.storm;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.Map;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichSpout;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
@SuppressWarnings("serial")
public class LineReaderSpout implements IRichSpout {
private SpoutOutputCollector collector;
private FileReader fileReader;
private boolean completed = false;
private TopologyContext context;
@Override
public void open(Map conf, TopologyContext context,
SpoutOutputCollector collector) {
try {
this.context = context;
this.fileReader = new FileReader(conf.get("inputFile").toString());
} catch (FileNotFoundException e) {
throw new RuntimeException("Error reading file "
+ conf.get("inputFile"));
}
this.collector = collector;
}
@Override
public void nextTuple() {
if (completed) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
String str;
BufferedReader reader = new BufferedReader(fileReader);
try {
while ((str = reader.readLine()) != null) {
this.collector.emit(new Values(str), str);
}
} catch (Exception e) {
throw new RuntimeException("Error reading typle", e);
} finally {
completed = true;
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
@Override
public void close() {
try {
fileReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public boolean isDistributed() {
return false;
}
@Override
public void activate() {
}
@Override
public void deactivate() {
}
@Override
public void ack(Object msgId) {
}
@Override
public void fail(Object msgId) {
}
@Override
public Map<String, Object> getComponentConfiguration() {
return null;
}
}
错误显示
jender:Storm$ java -cp target/Storm-1.0-SNAPSHOT.jar com.ender.storm.HelloStorm test.txt
Exception in thread "main" java.lang.NoClassDefFoundError: backtype/storm/topology/IRichSpout
at java.lang.Class.getDeclaredMethods0(Native Method)
at java.lang.Class.privateGetDeclaredMethods(Class.java:2615)
at java.lang.Class.getMethod0(Class.java:2856)
at java.lang.Class.getMethod(Class.java:1668)
at sun.launcher.LauncherHelper.getMainMethod(LauncherHelper.java:494)
at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:486)
Caused by: java.lang.ClassNotFoundException: backtype.storm.topology.IRichSpout
at java.net.URLClassLoader.run(URLClassLoader.java:366)
at java.net.URLClassLoader.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 6 more
Maven版本：
Apache Maven 3.0.5
Maven home: /usr/share/maven
Java version: 1.7.0_75, vendor: Oracle Corporation
Java home: /usr/lib/jvm/java-7-openjdk-i386/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "3.13.0-44-generic", arch: "i386", family: "unix"
Pom.xml:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ender.storm</groupId>
<artifactId>Storm</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<name>Storm</name>
<url>http://maven.apache.org</url>
<build>
<pluginManagement>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<compilerArgument></compilerArgument>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>0.9.1-incubating</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.6.0</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
'mvn package' 创建的 jar 文件包含：
META-INF/
META-INF/MANIFEST.MF
com/
com/ender/
com/ender/storm/
com/ender/storm/App.class
com/ender/storm/HelloStorm.class
com/ender/storm/RedisPubSubSpout$ListenerThread.class
com/ender/storm/RedisPubSubSpout$ListenerThread.class
com/ender/storm/LineReaderSpout.class
com/ender/storm/WordSpitterBolt.class
com/ender/storm/RedisPubSubSpout.class
com/ender/storm/WordCounterBolt.class
META-INF/maven/
META-INF/maven/com.ender.storm/
META-INF/maven/com.ender.storm/Storm/
META-INF/maven/com.ender.storm/Storm/pom.xml
META-INF/maven/com.ender.storm/Storm/pom.properties
它在 Ubuntu 14.04 下编译良好，这让我认为找到了所需的依赖项，但 apache-storm jar 不包含在 maven 创建的 jar 中。我该如何解决这个问题？
看起来你的 jar 没有包含任何依赖项，通常 jar 只包含项目的 类，而不是依赖项。
因此您需要将它们添加到类路径中，或者创建包含所有依赖项的大 jar。
要创建大 jar，请查看 Maven Assembly plugin（目标 - 具有依赖项的 jar）
Use jar-with-dependencies as the descriptorRef of your assembly-plugin
configuration in order to create a JAR which contains the binary
output of your project, along its the unpacked dependencies. This
built-in descriptor produces an assembly with the classifier
jar-with-dependencies using the JAR archive format.
或在 Maven Shade plugin 为此，您需要向 pom.xml 添加如下内容：（但请先阅读 shade 插件手册）
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>junit:junit</exclude>
</excludes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
我在运行时遇到问题，当我的应用程序在编译时找不到在 Maven 中列为依赖项的 类。应用程序编译没有问题。
申请如下
package com.ender.storm;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
public class HelloStorm {
public static void main(String[] args) throws Exception{
Config config = new Config();
config.put("inputFile", args[0]);
config.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
config.put(Config.TOPOLOGY_DEBUG, false);
config.setDebug(false);
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("line-reader-spout", new LineReaderSpout());
builder.setBolt("word-spitter", new WordSpitterBolt()).shuffleGrouping("line-reader-spout");
builder.setBolt("word-counter", new WordCounterBolt()).shuffleGrouping("word-spitter");
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("HelloStorm", config, builder.createTopology());
Thread.sleep(10000);
cluster.shutdown();
}
}
LineReaderSpout 如下所示
包 com.ender.storm;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.Map;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichSpout;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
@SuppressWarnings("serial")
public class LineReaderSpout implements IRichSpout {
private SpoutOutputCollector collector;
private FileReader fileReader;
private boolean completed = false;
private TopologyContext context;
@Override
public void open(Map conf, TopologyContext context,
SpoutOutputCollector collector) {
try {
this.context = context;
this.fileReader = new FileReader(conf.get("inputFile").toString());
} catch (FileNotFoundException e) {
throw new RuntimeException("Error reading file "
+ conf.get("inputFile"));
}
this.collector = collector;
}
@Override
public void nextTuple() {
if (completed) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
String str;
BufferedReader reader = new BufferedReader(fileReader);
try {
while ((str = reader.readLine()) != null) {
this.collector.emit(new Values(str), str);
}
} catch (Exception e) {
throw new RuntimeException("Error reading typle", e);
} finally {
completed = true;
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
@Override
public void close() {
try {
fileReader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public boolean isDistributed() {
return false;
}
@Override
public void activate() {
}
@Override
public void deactivate() {
}
@Override
public void ack(Object msgId) {
}
@Override
public void fail(Object msgId) {
}
@Override
public Map<String, Object> getComponentConfiguration() {
return null;
}
}
错误显示
jender:Storm$ java -cp target/Storm-1.0-SNAPSHOT.jar com.ender.storm.HelloStorm test.txt
Exception in thread "main" java.lang.NoClassDefFoundError: backtype/storm/topology/IRichSpout
at java.lang.Class.getDeclaredMethods0(Native Method)
at java.lang.Class.privateGetDeclaredMethods(Class.java:2615)
at java.lang.Class.getMethod0(Class.java:2856)
at java.lang.Class.getMethod(Class.java:1668)
at sun.launcher.LauncherHelper.getMainMethod(LauncherHelper.java:494)
at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:486)
Caused by: java.lang.ClassNotFoundException: backtype.storm.topology.IRichSpout
at java.net.URLClassLoader.run(URLClassLoader.java:366)
at java.net.URLClassLoader.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 6 more
Maven版本：
Apache Maven 3.0.5
Maven home: /usr/share/maven
Java version: 1.7.0_75, vendor: Oracle Corporation
Java home: /usr/lib/jvm/java-7-openjdk-i386/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "3.13.0-44-generic", arch: "i386", family: "unix"
Pom.xml:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ender.storm</groupId>
<artifactId>Storm</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<name>Storm</name>
<url>http://maven.apache.org</url>
<build>
<pluginManagement>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<compilerArgument></compilerArgument>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>0.9.1-incubating</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.6.0</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
'mvn package' 创建的 jar 文件包含：
META-INF/
META-INF/MANIFEST.MF
com/
com/ender/
com/ender/storm/
com/ender/storm/App.class
com/ender/storm/HelloStorm.class
com/ender/storm/RedisPubSubSpout$ListenerThread.class
com/ender/storm/RedisPubSubSpout$ListenerThread.class
com/ender/storm/LineReaderSpout.class
com/ender/storm/WordSpitterBolt.class
com/ender/storm/RedisPubSubSpout.class
com/ender/storm/WordCounterBolt.class
META-INF/maven/
META-INF/maven/com.ender.storm/
META-INF/maven/com.ender.storm/Storm/
META-INF/maven/com.ender.storm/Storm/pom.xml
META-INF/maven/com.ender.storm/Storm/pom.properties
它在 Ubuntu 14.04 下编译良好，这让我认为找到了所需的依赖项，但 apache-storm jar 不包含在 maven 创建的 jar 中。我该如何解决这个问题？
看起来你的 jar 没有包含任何依赖项，通常 jar 只包含项目的 类，而不是依赖项。
因此您需要将它们添加到类路径中，或者创建包含所有依赖项的大 jar。
要创建大 jar，请查看 Maven Assembly plugin（目标 - 具有依赖项的 jar）
Use jar-with-dependencies as the descriptorRef of your assembly-plugin configuration in order to create a JAR which contains the binary output of your project, along its the unpacked dependencies. This built-in descriptor produces an assembly with the classifier jar-with-dependencies using the JAR archive format.
或在 Maven Shade plugin 为此，您需要向 pom.xml 添加如下内容：（但请先阅读 shade 插件手册）
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>junit:junit</exclude>
</excludes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>