Consistent NPE (SEVERE error) in KeyValueLocator when getting row result from view

#1

When iterating over a row result from the the list of ViewResult objects in a 4 node Couchbase 3.0.0 cluster from the Java API (both 2.0.0 and 2.0.1-SNAPSHOT, although anecdotally, 2.0.0 appears more stable, yet still does this a lot), I am getting the following error. It happens quite frequently, but not consistently on the same record, yet is easily replicable, but seems to happen randomly on which record, but happens a lot.

Does not seem to occur in a non-cluster environment, only against my cluster of 4 servers - no issues against 1.

SEVERE: Exception processing: 262 com.couchbase.client.core.RequestEvent@7e1d66f1
java.lang.NullPointerException
at com.couchbase.client.core.node.locate.KeyValueLocator.locateForCouchbaseBucket(KeyValueLocator.java:90)
at com.couchbase.client.core.node.locate.KeyValueLocator.locate(KeyValueLocator.java:68)
at com.couchbase.client.core.RequestHandler.onEvent(RequestHandler.java:148)
at com.couchbase.client.core.RequestHandler.onEvent(RequestHandler.java:68)
at com.couchbase.client.deps.com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:128)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at com.couchbase.client.deps.io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
at java.lang.Thread.run(Thread.java:744)

Exception in thread “cb-core-3-2” java.lang.RuntimeException: java.lang.NullPointerException
at com.couchbase.client.deps.com.lmax.disruptor.FatalExceptionHandler.handleEventException(FatalExceptionHandler.java:45)
at com.couchbase.client.deps.com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:147)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at com.couchbase.client.deps.io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.NullPointerException
at com.couchbase.client.core.node.locate.KeyValueLocator.locateForCouchbaseBucket(KeyValueLocator.java:90)
at com.couchbase.client.core.node.locate.KeyValueLocator.locate(KeyValueLocator.java:68)
at com.couchbase.client.core.RequestHandler.onEvent(RequestHandler.java:148)
at com.couchbase.client.core.RequestHandler.onEvent(RequestHandler.java:68)
at com.couchbase.client.deps.com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:128)
… 4 more
java.lang.RuntimeException: java.util.concurrent.TimeoutException
at rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:481)
at rx.observables.BlockingObservable.singleOrDefault(BlockingObservable.java:382)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:43)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:38)
at com.voltaireapp.juryreportcreator.JuryReporter.resolveVoterRegRecordsForVTPerson(JuryReporter.java:167)
at com.voltaireapp.juryreportcreator.JuryReporter.resolveVTPerson(JuryReporter.java:198)
at com.voltaireapp.juryreportcreator.JuryReporter.runJuryReport(JuryReporter.java:243)
at com.voltaireapp.changeslistener.ChangesListener.router(ChangesListener.java:51)
at com.voltaireapp.changeslistener.ChangesListener.runChangesListener(ChangesListener.java:91)
at com.voltaireapp.changeslistener.ChangesListener.main(ChangesListener.java:135)
Caused by: java.util.concurrent.TimeoutException
at rx.internal.operators.OperatorTimeoutBase$TimeoutSubscriber.onTimeout(OperatorTimeoutBase.java:169)
at rx.internal.operators.OperatorTimeout$1$1.call(OperatorTimeout.java:42)
at rx.internal.schedulers.ScheduledAction.run(ScheduledAction.java:43)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Exception in thread “main” java.lang.RuntimeException: java.lang.RuntimeException: java.util.concurrent.TimeoutException
at com.voltaireapp.changeslistener.ChangesListener.runChangesListener(ChangesListener.java:105)
at com.voltaireapp.changeslistener.ChangesListener.main(ChangesListener.java:135)
Caused by: java.lang.RuntimeException: java.util.concurrent.TimeoutException
at rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:481)
at rx.observables.BlockingObservable.singleOrDefault(BlockingObservable.java:382)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:43)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:38)
at com.voltaireapp.juryreportcreator.JuryReporter.resolveVoterRegRecordsForVTPerson(JuryReporter.java:167)
at com.voltaireapp.juryreportcreator.JuryReporter.resolveVTPerson(JuryReporter.java:198)
at com.voltaireapp.juryreportcreator.JuryReporter.runJuryReport(JuryReporter.java:243)
at com.voltaireapp.changeslistener.ChangesListener.router(ChangesListener.java:51)
at com.voltaireapp.changeslistener.ChangesListener.runChangesListener(ChangesListener.java:91)
… 1 more
Caused by: java.util.concurrent.TimeoutException
at rx.internal.operators.OperatorTimeoutBase$TimeoutSubscriber.onTimeout(OperatorTimeoutBase.java:169)
at rx.internal.operators.OperatorTimeout$1$1.call(OperatorTimeout.java:42)
at rx.internal.schedulers.ScheduledAction.run(ScheduledAction.java:43)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)

#2

Beginning to suspect there is a reverse DNS lookup issue here (e.g. hostname resolves via public DNS to external ip, but reverse lookups on external IP get another hostname - a common thing on AWS and SoftLayer).

I understand this is a requirement/limitation of the architecture to have consistent/idempotent lookups/reverse lookups, so have reconfiged cluster to be no hostnames, just IPs and things seem to be working.

Perhaps a more descriptive error message might be a bit more useful if there’s a DNS mismatch/failure to find a server/some spurious string is showing up that isn’t in the cluster list to point to this. I am relatively sure this is what is going on. Probably not a bug per se, but the error message/failure mode perhaps could point to the issue a bit more strongly?

#3

Well, it there are still errors, but they are different, when reading a property off a value. Same behavior and pattern as above issue, but different error even on a cluster with copacetic DNS settings.

java.lang.RuntimeException: java.util.concurrent.TimeoutException
at rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:481)
at rx.observables.BlockingObservable.singleOrDefault(BlockingObservable.java:382)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:47)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:38)
at com.voltaireapp.changeslistener.ChangesListener.runChangesListener(ChangesListener.java:86)
at com.voltaireapp.changeslistener.ChangesListener.main(ChangesListener.java:135)
Caused by: java.util.concurrent.TimeoutException
at rx.internal.operators.OperatorTimeoutBase$TimeoutSubscriber.onTimeout(OperatorTimeoutBase.java:169)
at rx.internal.operators.OperatorTimeout$1$1.call(OperatorTimeout.java:42)
at rx.internal.schedulers.ScheduledAction.run(ScheduledAction.java:43)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Exception in thread “main” java.lang.RuntimeException: java.lang.RuntimeException: java.util.concurrent.TimeoutException
at com.voltaireapp.changeslistener.ChangesListener.runChangesListener(ChangesListener.java:105)
at com.voltaireapp.changeslistener.ChangesListener.main(ChangesListener.java:135)
Caused by: java.lang.RuntimeException: java.util.concurrent.TimeoutException
at rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:481)
at rx.observables.BlockingObservable.singleOrDefault(BlockingObservable.java:382)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:47)
at com.couchbase.client.java.view.DefaultViewRow.document(DefaultViewRow.java:38)
at com.voltaireapp.changeslistener.ChangesListener.runChangesListener(ChangesListener.java:86)
… 1 more
Caused by: java.util.concurrent.TimeoutException
at rx.internal.operators.OperatorTimeoutBase$TimeoutSubscriber.onTimeout(OperatorTimeoutBase.java:169)
at rx.internal.operators.OperatorTimeout$1$1.call(OperatorTimeout.java:42)
at rx.internal.schedulers.ScheduledAction.run(ScheduledAction.java:43)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)

#4

Thanks, I’ll look into that as well.

#5

Can you try this 2.0.1-pre version and see if it works for you?

https://dl.dropboxusercontent.com/u/10007675/Couchbase-Java-Client-2.0.1-SNAPSHOT-b0921ce.zip