Added an option to enable dropping of new points written after action queue exhaustion. (#689)

simarpreets · web-flow · commit f590c59c7aee · 2020-07-20T07:12:53.000+02:00
1. Added an option to drop the points being written after BatchProcessor#queue is exhausted.
2. Added an option to plug in a handler which will be invoked if above happens.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## 2.20 [unreleased]
 
+### Features
+- Add an option in `BatchOption` to prevent `InfluxDB#write` from blocking when actions queue is exhausted. [Issue #668](https://github.com/influxdata/influxdb-java/issues/688)
+
+
 ## 2.19 [2020-05-18]
 
 ## 2.18 [2020-04-17]
diff --git a/MANUAL.md b/MANUAL.md
@@ -101,7 +101,18 @@ It's possible to use both approaches at the same time: set a default database us
   public static final int DEFAULT_JITTER_INTERVAL_DURATION = 0;
   public static final int DEFAULT_BUFFER_LIMIT = 10000;
   public static final TimeUnit DEFAULT_PRECISION = TimeUnit.NANOSECONDS;
+  public static final boolean DEFAULT_DROP_ACTIONS_ON_QUEUE_EXHAUSTION = false;
 ```
+#### Configuring behaviour of batch writes when the action queue exhausts
+With batching enabled, the client provides two options on how to deal with **action queue** (where the points are accumulated as a batch) exhaustion.
+1. When `dropActionsOnQueueExhaustion` is `false` (default value), `InfluxDB#write` will be blocked till the space is created in the action queue.
+2. When `dropActionsOnQueueExhaustion` is `true`, new writes using `InfluxDB#write` will dropped and `droppedActionHandler` will be called.
+   Example usage:
+   ```Java
+   influxDB.enableBatch(BatchOptions.DEFAULTS.dropActionsOnQueueExhaustion(true)
+                                             .droppedActionHandler((point) -> log.error("Point dropped due to action queue exhaustion.")));
+   ```
+    
 
 #### Configuring the jitter interval for batch writes
 
diff --git a/src/main/java/org/influxdb/BatchOptions.java b/src/main/java/org/influxdb/BatchOptions.java
@@ -6,6 +6,7 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 
 /**
  * BatchOptions are used to configure batching of individual data point writes
@@ -19,6 +20,8 @@ public final class BatchOptions implements Cloneable {
   public static final int DEFAULT_JITTER_INTERVAL_DURATION = 0;
   public static final int DEFAULT_BUFFER_LIMIT = 10000;
   public static final TimeUnit DEFAULT_PRECISION = TimeUnit.NANOSECONDS;
+  public static final boolean DEFAULT_DROP_ACTIONS_ON_QUEUE_EXHAUSTION = false;
+
 
   /**
    * Default batch options. This class is immutable, each configuration
@@ -32,10 +35,14 @@ public final class BatchOptions implements Cloneable {
   private int jitterDuration = DEFAULT_JITTER_INTERVAL_DURATION;
   private int bufferLimit = DEFAULT_BUFFER_LIMIT;
   private TimeUnit precision = DEFAULT_PRECISION;
+  private boolean dropActionsOnQueueExhaustion = DEFAULT_DROP_ACTIONS_ON_QUEUE_EXHAUSTION;
+  private Consumer<Point> droppedActionHandler = (point) -> {
+  };
 
   private ThreadFactory threadFactory = Executors.defaultThreadFactory();
   BiConsumer<Iterable<Point>, Throwable> exceptionHandler = (points, throwable) -> {
   };
+
   private InfluxDB.ConsistencyLevel consistency = InfluxDB.ConsistencyLevel.ONE;
 
   private BatchOptions() {
@@ -133,6 +140,33 @@ public BatchOptions precision(final TimeUnit precision) {
     return clone;
   }
 
+  /**
+   * Set to define the behaviour when the action queue exhausts. If unspecified, will default to false which means
+   * that the {@link InfluxDB#write(Point)} will be blocked till the space in the queue is created.
+   * true means that the newer actions being written to the queue will be dropped and
+   * {@link BatchOptions#droppedActionHandler} will be called.
+   * @param dropActionsOnQueueExhaustion sets the behavior
+   * @return the BatchOptions instance to be able to use it in a fluent manner.
+   */
+  public BatchOptions dropActionsOnQueueExhaustion(final boolean dropActionsOnQueueExhaustion) {
+    BatchOptions clone = getClone();
+    clone.dropActionsOnQueueExhaustion = dropActionsOnQueueExhaustion;
+    return clone;
+  }
+
+  /**
+   * Handler to handle dropped actions due to queue actions. This is only valid when
+   * {@link BatchOptions#dropActionsOnQueueExhaustion} is set to true.
+   * @param droppedActionHandler to handle action drops on action queue exhaustion.
+   * @return the BatchOptions instance to be able to use it in a fluent manner.
+   */
+  public BatchOptions droppedActionHandler(final Consumer<Point> droppedActionHandler) {
+    BatchOptions clone = getClone();
+    clone.droppedActionHandler = droppedActionHandler;
+    return clone;
+  }
+
+
   /**
    * @return actions the number of actions to collect
    */
@@ -190,6 +224,21 @@ public TimeUnit getPrecision() {
     return precision;
   }
 
+
+  /**
+   * @return a boolean determining whether to drop actions on action queue exhaustion.
+   */
+  public boolean isDropActionsOnQueueExhaustion() {
+    return dropActionsOnQueueExhaustion;
+  }
+
+  /**
+   * @return a consumer function to handle actions drops on action queue exhaustion.
+   */
+  public Consumer<Point> getDroppedActionHandler() {
+    return droppedActionHandler;
+  }
+
   private BatchOptions getClone() {
     try {
       return (BatchOptions) this.clone();
diff --git a/src/main/java/org/influxdb/impl/BatchProcessor.java b/src/main/java/org/influxdb/impl/BatchProcessor.java
@@ -19,6 +19,7 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -43,6 +44,8 @@ public final class BatchProcessor {
   private final int jitterInterval;
   private final TimeUnit precision;
   private final BatchWriter batchWriter;
+  private boolean dropActionsOnQueueExhaustion;
+  Consumer<Point> droppedActionHandler;
 
   /**
    * The Builder to create a BatchProcessor instance.
@@ -62,6 +65,8 @@ public static final class Builder {
     private BiConsumer<Iterable<Point>, Throwable> exceptionHandler = (entries, throwable) -> { };
     private ConsistencyLevel consistencyLevel;
 
+    private boolean dropActionsOnQueueExhaustion;
+    private Consumer<Point> droppedActionsHandler;
     /**
      * @param threadFactory
      *            is optional.
@@ -152,6 +157,37 @@ public Builder exceptionHandler(final BiConsumer<Iterable<Point>, Throwable> han
       return this;
     }
 
+    /**
+     * To define the behaviour when the action queue exhausts. If unspecified, will default to false which means that
+     * the {@link InfluxDB#write(Point)} will be blocked till the space in the queue is created.
+     * true means that the newer actions being written to the queue will dropped and
+     * {@link BatchProcessor#droppedActionHandler} will be called.
+     *
+     * @param dropActionsOnQueueExhaustion
+     *            the dropActionsOnQueueExhaustion
+     *
+     * @return this Builder to use it fluent
+     */
+    public Builder dropActionsOnQueueExhaustion(final boolean dropActionsOnQueueExhaustion) {
+      this.dropActionsOnQueueExhaustion = dropActionsOnQueueExhaustion;
+      return this;
+    }
+
+    /**
+     * A callback to be used when an actions are dropped on action queue exhaustion.
+     *
+     * @param handler
+     *            the handler
+     *
+     * @return this Builder to use it fluent
+     */
+    public Builder droppedActionHandler(final Consumer<Point> handler) {
+      this.droppedActionsHandler = handler;
+      return this;
+    }
+
+
+
     /**
      * Consistency level for batch write.
      *
@@ -200,7 +236,7 @@ public BatchProcessor build() {
       }
       return new BatchProcessor(this.influxDB, batchWriter, this.threadFactory, this.actions, this.flushIntervalUnit,
                                 this.flushInterval, this.jitterInterval, exceptionHandler, this.consistencyLevel,
-                                this.precision);
+                                this.precision, this.dropActionsOnQueueExhaustion, this.droppedActionsHandler);
     }
   }
 
@@ -262,7 +298,8 @@ public static Builder builder(final InfluxDB influxDB) {
   BatchProcessor(final InfluxDB influxDB, final BatchWriter batchWriter, final ThreadFactory threadFactory,
                  final int actions, final TimeUnit flushIntervalUnit, final int flushInterval, final int jitterInterval,
                  final BiConsumer<Iterable<Point>, Throwable> exceptionHandler,
-                 final ConsistencyLevel consistencyLevel, final TimeUnit precision) {
+                 final ConsistencyLevel consistencyLevel, final TimeUnit precision,
+                 final boolean dropActionsOnQueueExhaustion, final Consumer<Point> droppedActionHandler) {
     super();
     this.influxDB = influxDB;
     this.batchWriter = batchWriter;
@@ -274,6 +311,8 @@ public static Builder builder(final InfluxDB influxDB) {
     this.exceptionHandler = exceptionHandler;
     this.consistencyLevel = consistencyLevel;
     this.precision = precision;
+    this.dropActionsOnQueueExhaustion = dropActionsOnQueueExhaustion;
+    this.droppedActionHandler = droppedActionHandler;
     if (actions > 1 && actions < Integer.MAX_VALUE) {
         this.queue = new LinkedBlockingQueue<>(actions);
     } else {
@@ -359,7 +398,14 @@ void write() {
    */
   void put(final AbstractBatchEntry batchEntry) {
     try {
-        this.queue.put(batchEntry);
+        if (this.dropActionsOnQueueExhaustion) {
+          if (!this.queue.offer(batchEntry)) {
+            this.droppedActionHandler.accept(batchEntry.getPoint());
+            return;
+          }
+        } else {
+          this.queue.put(batchEntry);
+        }
     } catch (InterruptedException e) {
         throw new RuntimeException(e);
     }
@@ -403,4 +449,11 @@ BatchWriter getBatchWriter() {
     return batchWriter;
   }
 
+  public boolean isDropActionsOnQueueExhaustion() {
+    return dropActionsOnQueueExhaustion;
+  }
+
+  public Consumer<Point> getDroppedActionHandler() {
+    return droppedActionHandler;
+  }
 }
diff --git a/src/main/java/org/influxdb/impl/InfluxDBImpl.java b/src/main/java/org/influxdb/impl/InfluxDBImpl.java
@@ -310,6 +310,8 @@ public InfluxDB enableBatch(final BatchOptions batchOptions) {
             .bufferLimit(batchOptions.getBufferLimit())
             .consistencyLevel(batchOptions.getConsistency())
             .precision(batchOptions.getPrecision())
+            .dropActionsOnQueueExhaustion(batchOptions.isDropActionsOnQueueExhaustion())
+            .droppedActionHandler(batchOptions.getDroppedActionHandler())
             .build();
     this.batchEnabled.set(true);
     return this;
@@ -343,13 +345,14 @@ public InfluxDB enableBatch(final int actions, final int flushDuration, final Ti
   public InfluxDB enableBatch(final int actions, final int flushDuration, final TimeUnit flushDurationTimeUnit,
                               final ThreadFactory threadFactory,
                               final BiConsumer<Iterable<Point>, Throwable> exceptionHandler) {
-    enableBatch(actions, flushDuration, 0, flushDurationTimeUnit, threadFactory, exceptionHandler);
+    enableBatch(actions, flushDuration, 0, flushDurationTimeUnit, threadFactory, exceptionHandler, false, null);
     return this;
   }
 
   private InfluxDB enableBatch(final int actions, final int flushDuration, final int jitterDuration,
                                final TimeUnit durationTimeUnit, final ThreadFactory threadFactory,
-                               final BiConsumer<Iterable<Point>, Throwable> exceptionHandler) {
+                               final BiConsumer<Iterable<Point>, Throwable> exceptionHandler,
+                               final boolean dropActionsOnQueueExhaustion, final Consumer<Point> droppedActionHandler) {
     if (this.batchEnabled.get()) {
       throw new IllegalStateException("BatchProcessing is already enabled.");
     }
@@ -360,6 +363,8 @@ private InfluxDB enableBatch(final int actions, final int flushDuration, final i
             .interval(flushDuration, jitterDuration, durationTimeUnit)
             .threadFactory(threadFactory)
             .consistencyLevel(consistency)
+            .dropActionsOnQueueExhaustion(dropActionsOnQueueExhaustion)
+            .droppedActionHandler(droppedActionHandler)
             .build();
     this.batchEnabled.set(true);
     return this;
diff --git a/src/test/java/org/influxdb/BatchOptionsTest.java b/src/test/java/org/influxdb/BatchOptionsTest.java
diff --git a/src/test/java/org/influxdb/impl/BatchProcessorTest.java b/src/test/java/org/influxdb/impl/BatchProcessorTest.java
