Nutch 爬行给出错误 "Error from server at http://localhost:8983/solr/nutch: java.lang.NullPointerException"
Nutch crawling giving error "Error from server at http://localhost:8983/solr/nutch: java.lang.NullPointerException"
我正在尝试抓取网站并使用 Solr 对其编制索引。我有一些使用 UpdateRequestProcessor 更新的自定义字段。我的自定义 URP 如下所示：
FieldProcessorFactory.java
package ved;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
public class FieldProcessorFactory extends UpdateRequestProcessorFactory
{
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next)
{
return new FieldProcessor(next);
}
}
FieldProcessor.java
package ved;
import java.io.IOException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import java.util.StringTokenizer;
class FieldProcessor extends UpdateRequestProcessor
{
public FieldProcessor( UpdateRequestProcessor next) {
super( next );
}
public static int countWords(String arg) {
if (arg == null) {
return 0;
}
return arg.split("[\pP\s&&[^']]+").length;
}
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
String title = (String) cmd.getSolrInputDocument().get("title").getValue();
int title_length_i = countWords(title);
cmd.getSolrInputDocument().addField("title_length_i", title_length_i);
String anchor = (String) cmd.getSolrInputDocument().get("anchor").getValue();
int anchor_length_i = countWords(anchor);
cmd.getSolrInputDocument().addField("anchor_length_i", anchor_length_i);
String body = (String) cmd.getSolrInputDocument().get("content").getValue();
int body_length_i = countWords(body);
cmd.getSolrInputDocument().addField("body_length_i", body_length_i);
String url = (String) cmd.getSolrInputDocument().get("url").getValue();
int url_length_i = countWords(url);
cmd.getSolrInputDocument().addField("url_length_i", url_length_i);
int whole_document_length_i = title_length_i + body_length_i + url_length_i;
cmd.getSolrInputDocument().addField("whole_document_length_i", whole_document_length_i);
super.processAdd(cmd);
}
}
schema.xml
<field name="title_length_i" type="int" stored="true" indexed="true"/>
<field name="anchor_length_i" type="int" stored="true" indexed="true"/>
<field name="body_length_i" type="int" stored="true" indexed="true"/>
<field name="url_length_i" type="int" stored="true" indexed="true"/>
<field name="whole_document_length_i" type="int" stored="true" indexed="true"/>
solrconfig.xml
<lib dir="${solr.install.dir:../../../..}/plugins/" regex="fieldProcessor.jar" />
<updateRequestProcessorChain name="process-articles" lib="custom-libs" version="1">
<processor class="ved.FieldProcessorFactory"/>
<processor class="solr.LogUpdateProcessorFactory" />
<processor class="solr.DistributedUpdateProcessorFactory" />
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
<lst name="defaults">
<str name="df">text</str>
<str name="update.chain">process-articles</str>
</lst>
</initParams>
但在抓取过程中出现以下错误：
java.lang.Exception: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://localhost:8983/solr/nutch: java.lang.NullPointerException
at ved.FieldProcessor.processAdd(FieldProcessor.java:29)
at org.apache.solr.handler.loader.JavabinLoader.update(JavabinLoader.java:98)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:180)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readIterator(JavaBinUpdateRequestCodec.java:136)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:306)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readNamedList(JavaBinUpdateRequestCodec.java:122)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:271)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:173)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:187)
at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
at org.apache.solr.handler.UpdateRequestHandler.load(UpdateRequestHandler.java:97)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2477)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:723)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:529)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://localhost:8983/solr/nutch: java.lang.NullPointerException
at ved.FieldProcessor.processAdd(FieldProcessor.java:29)
at org.apache.solr.handler.loader.JavabinLoader.update(JavabinLoader.java:98)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:180)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readIterator(JavaBinUpdateRequestCodec.java:136)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:306)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readNamedList(JavaBinUpdateRequestCodec.java:122)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:271)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:173)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:187)
at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
at org.apache.solr.handler.UpdateRequestHandler.load(UpdateRequestHandler.java:97)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2477)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:723)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:529)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:240)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:229)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.nutch.indexwriter.solr.SolrIndexWriter.push(SolrIndexWriter.java:210)
at org.apache.nutch.indexwriter.solr.SolrIndexWriter.commit(SolrIndexWriter.java:188)
at org.apache.nutch.indexwriter.solr.SolrIndexWriter.close(SolrIndexWriter.java:179)
at org.apache.nutch.indexer.IndexWriters.close(IndexWriters.java:117)
at org.apache.nutch.indexer.IndexerOutputFormat.close(IndexerOutputFormat.java:44)
at org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter.close(ReduceTask.java:502)
at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:456)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:392)
at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2020-02-22 14:31:49,730 ERROR indexer.IndexingJob - Indexer: java.io.IOException: Job failed!
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:873)
at org.apache.nutch.indexer.IndexingJob.index(IndexingJob.java:147)
at org.apache.nutch.indexer.IndexingJob.run(IndexingJob.java:230)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.nutch.indexer.IndexingJob.main(IndexingJob.java:239)
可能出了什么问题？
我不是很确定，如果您能说出您是如何配置 nutch 来抓取数据或让我们说出 url，那可能是这样。我看到可能发生的一个问题是在 processAdd 方法中，您没有向 Solr 文档添加 Id 字段，因为它是在抓取后区分 solr 索引中的每个文档的必填字段。即使在 schema.xml 中，我也没有看到任何 id 字段，所以我想知道索引将如何解释已爬网的文件。
如果 FieldProcessor.java 文件中的行数与异常中的行数匹配，这些行似乎来自：
String anchor = (String) cmd.getSolrInputDocument().get("anchor").getValue();
我可以看到 get(key) 方法 can return a null value 如果请求的字段不存在于 SolrInputDocument 实例中。您应该检查：
cmd.getSolrInputDocument().get("anchor")
returns 非空值。
根据我使用 Nutch 的经验，anchor 字段并不总是出现在已抓取的文档中。此外，请记住，如果在模式中声明了一个字段并不一定意味着所有文档都将具有该字段（除非根据需要指定，在这种情况下，如果它不存在，则该文档将被 Solr 拒绝早期阶段）。
我正在尝试抓取网站并使用 Solr 对其编制索引。我有一些使用 UpdateRequestProcessor 更新的自定义字段。我的自定义 URP 如下所示：
FieldProcessorFactory.java
package ved;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
public class FieldProcessorFactory extends UpdateRequestProcessorFactory
{
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next)
{
return new FieldProcessor(next);
}
}
FieldProcessor.java
package ved;
import java.io.IOException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import java.util.StringTokenizer;
class FieldProcessor extends UpdateRequestProcessor
{
public FieldProcessor( UpdateRequestProcessor next) {
super( next );
}
public static int countWords(String arg) {
if (arg == null) {
return 0;
}
return arg.split("[\pP\s&&[^']]+").length;
}
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
String title = (String) cmd.getSolrInputDocument().get("title").getValue();
int title_length_i = countWords(title);
cmd.getSolrInputDocument().addField("title_length_i", title_length_i);
String anchor = (String) cmd.getSolrInputDocument().get("anchor").getValue();
int anchor_length_i = countWords(anchor);
cmd.getSolrInputDocument().addField("anchor_length_i", anchor_length_i);
String body = (String) cmd.getSolrInputDocument().get("content").getValue();
int body_length_i = countWords(body);
cmd.getSolrInputDocument().addField("body_length_i", body_length_i);
String url = (String) cmd.getSolrInputDocument().get("url").getValue();
int url_length_i = countWords(url);
cmd.getSolrInputDocument().addField("url_length_i", url_length_i);
int whole_document_length_i = title_length_i + body_length_i + url_length_i;
cmd.getSolrInputDocument().addField("whole_document_length_i", whole_document_length_i);
super.processAdd(cmd);
}
}
schema.xml
<field name="title_length_i" type="int" stored="true" indexed="true"/>
<field name="anchor_length_i" type="int" stored="true" indexed="true"/>
<field name="body_length_i" type="int" stored="true" indexed="true"/>
<field name="url_length_i" type="int" stored="true" indexed="true"/>
<field name="whole_document_length_i" type="int" stored="true" indexed="true"/>
solrconfig.xml
<lib dir="${solr.install.dir:../../../..}/plugins/" regex="fieldProcessor.jar" />
<updateRequestProcessorChain name="process-articles" lib="custom-libs" version="1">
<processor class="ved.FieldProcessorFactory"/>
<processor class="solr.LogUpdateProcessorFactory" />
<processor class="solr.DistributedUpdateProcessorFactory" />
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse">
<lst name="defaults">
<str name="df">text</str>
<str name="update.chain">process-articles</str>
</lst>
</initParams>
但在抓取过程中出现以下错误：
java.lang.Exception: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://localhost:8983/solr/nutch: java.lang.NullPointerException
at ved.FieldProcessor.processAdd(FieldProcessor.java:29)
at org.apache.solr.handler.loader.JavabinLoader.update(JavabinLoader.java:98)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:180)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readIterator(JavaBinUpdateRequestCodec.java:136)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:306)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readNamedList(JavaBinUpdateRequestCodec.java:122)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:271)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:173)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:187)
at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
at org.apache.solr.handler.UpdateRequestHandler.load(UpdateRequestHandler.java:97)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2477)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:723)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:529)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://localhost:8983/solr/nutch: java.lang.NullPointerException
at ved.FieldProcessor.processAdd(FieldProcessor.java:29)
at org.apache.solr.handler.loader.JavabinLoader.update(JavabinLoader.java:98)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:180)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readIterator(JavaBinUpdateRequestCodec.java:136)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:306)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.readNamedList(JavaBinUpdateRequestCodec.java:122)
at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:271)
at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:251)
at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:173)
at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:187)
at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
at org.apache.solr.handler.UpdateRequestHandler.load(UpdateRequestHandler.java:97)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2477)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:723)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:529)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:240)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:229)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.nutch.indexwriter.solr.SolrIndexWriter.push(SolrIndexWriter.java:210)
at org.apache.nutch.indexwriter.solr.SolrIndexWriter.commit(SolrIndexWriter.java:188)
at org.apache.nutch.indexwriter.solr.SolrIndexWriter.close(SolrIndexWriter.java:179)
at org.apache.nutch.indexer.IndexWriters.close(IndexWriters.java:117)
at org.apache.nutch.indexer.IndexerOutputFormat.close(IndexerOutputFormat.java:44)
at org.apache.hadoop.mapred.ReduceTask$OldTrackingRecordWriter.close(ReduceTask.java:502)
at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:456)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:392)
at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2020-02-22 14:31:49,730 ERROR indexer.IndexingJob - Indexer: java.io.IOException: Job failed!
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:873)
at org.apache.nutch.indexer.IndexingJob.index(IndexingJob.java:147)
at org.apache.nutch.indexer.IndexingJob.run(IndexingJob.java:230)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.nutch.indexer.IndexingJob.main(IndexingJob.java:239)
可能出了什么问题？
我不是很确定，如果您能说出您是如何配置 nutch 来抓取数据或让我们说出 url，那可能是这样。我看到可能发生的一个问题是在 processAdd 方法中，您没有向 Solr 文档添加 Id 字段，因为它是在抓取后区分 solr 索引中的每个文档的必填字段。即使在 schema.xml 中，我也没有看到任何 id 字段，所以我想知道索引将如何解释已爬网的文件。
如果 FieldProcessor.java 文件中的行数与异常中的行数匹配，这些行似乎来自：
String anchor = (String) cmd.getSolrInputDocument().get("anchor").getValue();
我可以看到 get(key) 方法 can return a null value 如果请求的字段不存在于 SolrInputDocument 实例中。您应该检查：
cmd.getSolrInputDocument().get("anchor")
returns 非空值。
根据我使用 Nutch 的经验，anchor 字段并不总是出现在已抓取的文档中。此外，请记住，如果在模式中声明了一个字段并不一定意味着所有文档都将具有该字段（除非根据需要指定，在这种情况下，如果它不存在，则该文档将被 Solr 拒绝早期阶段）。