datastax · clun · Apr 4, 2025 · Nov 21, 2024 · Nov 30, 2024 · Nov 30, 2024
diff --git a/.github/workflows/ci-astra-col-db-dbadmin.yaml b/.github/workflows/ci-astra-col-db-dbadmin.yaml
@@ -16,7 +16,7 @@ on:
       java_version:
         description: 'JDK version to use'
         required: true
-        default: '11.0'
+        default: '17.0'
       cache_pkg_mgr_name:
         description: 'Cache package manager to be used'
         required: true

diff --git a/.github/workflows/ci-astra-dev-new.yml b/.github/workflows/ci-astra-dev-new.yml
@@ -17,7 +17,7 @@ on:
       java_version:
         description: 'JDK version to use'
         required: true
-        default: '11.0'
+        default: '17.0'
       cache_pkg_mgr_name:
         description: 'Cache package manager to be used'
         required: true
@@ -100,9 +100,9 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'adopt'
-          java-version: '11.0'
-          cache: 'maven'
+          distribution: ${{ inputs.java_distribution }}
+          java-version: ${{ inputs.java_version }}
+          cache: ${{ inputs.cache_pkg_mgr_name }}
 
       - name: Run Maven Tests
         env:

diff --git a/.github/workflows/ci-astra-vectorize-bedrock-hf.yaml b/.github/workflows/ci-astra-vectorize-bedrock-hf.yaml
@@ -24,7 +24,7 @@ on:
       java_version:
         description: 'JDK version to use'
         required: true
-        default: '11.0'
+        default: '17.0'
       cache_pkg_mgr_name:
         description: 'Cache package manager to be used'
         required: true

diff --git a/File b/File
diff --git a/Preferences... b/Preferences...
diff --git a/RELEASE.MD b/RELEASE.MD
@@ -15,64 +15,6 @@ docker-compose -f ./docker-compose-dse.yml up -d
 docker run -it --rm --network container:$(docker ps | grep dse | cut -b 1-12) cassandra:latest cqlsh -u cassandra -p cassandra
 ```
 
-
-```sql
-  insert into default_keyspace.table_all_returns (
-    p_ascii,
-    p_bigint,
-    p_blob,
-    p_boolean,
-    p_date,
-    p_decimal,
-    p_double,
-    p_double_minf,
-    p_double_pinf,
-    p_duration,
-    p_float,
-    p_float_nan,
-    p_inet,
-    p_int,
-    p_smallint,
-    p_text,
-    p_text_nulled,
-    p_time,
-    p_timestamp,
-    p_tinyint,
-    p_uuid,
-    p_varint,
-    p_vector,
-    p_list_int,
-    // p_map_text_int,
-    p_set_int
-) values (
-             'abc',
-             10000,
-             0xabcdef0123456789,
-             true,
-             '2015-05-03',
-             123.45,
-             987.6543,
-             -Infinity,
-             Infinity,
-             12h48m,
-             66.55,
-             NaN,
-             '12.34.56.78',
-             2024,
-             200,
-             'måndag',
-             null,
-             '13:30:54.234',
-             '2015-05-03 13:30:54.234',
-             104,
-             9c5b94b1-35ad-49bb-b118-8e8fc24abf80,
-             123456789,
-             [0.1, 0.2, 0.3],
-             [4, 17, 34],
-             // {'a': 101, 'b': 102},
-             {9, 81}
-         );
-```
 ## Run the Test
 
 ### Prerequisites

diff --git a/Resources b/Resources
diff --git a/TEST.MD b/TEST.MD
@@ -0,0 +1,52 @@
+
+## Run against Local HCD Instance 
+(_to run locally your must have the data api running locally on op of HCD or DSE_)
+
+```bash
+mvn clean test -Ptest_local
+```
+
+> **Why it works ?**
+> 
+> - In `pom.xml` the environment variable `ASTRA_DB_JAVA_TEST_ENV` is set to `local` 
+> - Tests are annotated with
+> 
+> ```java
+> @EnabledIfSystemProperty(named = "ASTRA_DB_JAVA_TEST_ENV", matches = "local")
+> @DisabledIfSystemProperty(named = "ASTRA_DB_JAVA_TEST_ENV", matches = "(?!local)")
+> public class MyTest {
+> }
+> ```
+
+- To run with IDE make sure to set the environment variable `ASTRA_DB_JAVA_TEST_ENV` to `local` in your Run configuration
+
+
+## Run against Astra DEV
+
+- `ASTRA_DB_APPLICATION_TOKEN_DEV` should be defined in your environment variables, it is not in the different POM for privacy reasons.
+
+- To run with Maven (AWS,`eu-west-2`)
+
+```bash
+mvn clean test -Ptest_astra_dev
+```
+
+- To run with Maven on One particular region
+
+```java
+mvn clean test -Ptest_astra_dev \
+  -DASTRA_CLOUD_PROVIDER_DEV=GCP \
+  -DASTRA_CLOUD_REGION_DEV=us-central-1
+```
+
+- To run overriding also the token
+
+```java
+mvn clean test \
+  -Ptest_astra_dev \
+  -DASTRA_CLOUD_PROVIDER_DEV=GCP \
+  -DASTRA_CLOUD_REGION_DEV=us-central-1 \
+  -ASTRA_DB_APPLICATION_TOKEN_DEV=AstraCS:... \
+  -Dtest="com.datastax.astra.test.integration.*.*Test"
+```
+
diff --git a/tools/pom.xml → astra-db-java-tools/pom.xml b/tools/pom.xml → astra-db-java-tools/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>com.datastax.astra</groupId>
         <artifactId>astra-db-java-parent</artifactId>
-        <version>2.0.0-SNAPSHOT</version>
+        <version>2.0.0-PREVIEW1-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/...stax/astra/tool/loader/csv/CsvLoader.java → ...stax/astra/tool/loader/csv/CsvLoader.java b/...stax/astra/tool/loader/csv/CsvLoader.java → ...stax/astra/tool/loader/csv/CsvLoader.java
@@ -1,7 +1,7 @@
 package com.datastax.astra.tool.loader.csv;
 
 import com.datastax.astra.client.collections.Collection;
-import com.datastax.astra.client.collections.documents.Document;
+import com.datastax.astra.client.collections.definition.documents.Document;
 import com.opencsv.CSVReader;
 import lombok.extern.slf4j.Slf4j;
 

diff --git a/...ra/tool/loader/csv/CsvLoaderSettings.java → ...ra/tool/loader/csv/CsvLoaderSettings.java b/...ra/tool/loader/csv/CsvLoaderSettings.java → ...ra/tool/loader/csv/CsvLoaderSettings.java
diff --git a/...x/astra/tool/loader/csv/CsvRowMapper.java → ...x/astra/tool/loader/csv/CsvRowMapper.java b/...x/astra/tool/loader/csv/CsvRowMapper.java → ...x/astra/tool/loader/csv/CsvRowMapper.java
@@ -1,6 +1,6 @@
 package com.datastax.astra.tool.loader.csv;
 
-import com.datastax.astra.client.collections.documents.Document;
+import com.datastax.astra.client.collections.definition.documents.Document;
 
 /**
  * Settings for the CSV Loader
@@ -13,6 +13,6 @@ public interface CsvRowMapper {
      * @param doc
      *      document to process
      */
-    Document map(Document doc) ;
+    Document map(Document doc);
 
 }
diff --git a/...a-db-java-tools/src/main/java/com/datastax/astra/tool/loader/json/JsonDocumentLoader.java b/...a-db-java-tools/src/main/java/com/datastax/astra/tool/loader/json/JsonDocumentLoader.java
@@ -0,0 +1,104 @@
+package com.datastax.astra.tool.loader.json;
+
+import com.datastax.astra.client.collections.Collection;
+import com.datastax.astra.client.collections.definition.documents.Document;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Slf4j
+public class JsonDocumentLoader {
+
+    /**
+     * Distributed import of CSV file into Astra.
+     *
+     * @throws Exception
+     *      exception in processing CSV
+     */
+    public static void load(String fileName, Collection<Document> collection, JsonRecordMapper processor) throws Exception {
+        load(fileName, JsonLoaderSettings.builder().build(), collection, processor);
+    }
+
+    /**
+     * Distributed import of CSV file into Astra.
+     *
+     * @throws Exception
+     *      exception in processing CSV
+     */
+    public static void load(String fileName, JsonLoaderSettings settings, Collection<Document> collection) throws Exception {
+        load(fileName, settings, collection, doc -> doc);
+    }
+
+    /**
+     * Distributed import of a JSON file into Astra.
+     *
+     * @param settings
+     *     settings
+     * @param collection
+     *     collection
+     * @param processor
+     *     processor
+     * @throws Exception
+     *      exception in processing CSV
+     */
+    public static void load(String fileName,
+                            JsonLoaderSettings settings,
+                            Collection<Document> collection,
+                            JsonRecordMapper processor)
+    throws Exception {
+        AtomicInteger counter = new AtomicInteger();
+        long top = System.currentTimeMillis();
+        // Multithreaded executor to process the CSV file
+        long startTime = System.currentTimeMillis();
+        ObjectMapper objectMapper = new ObjectMapper();
+        List<Document> batch = new ArrayList<>(settings.batchSize);
+        ExecutorService executor = Executors.newFixedThreadPool(settings.threadPoolSize);
+
+        try (JsonParser parser = new JsonFactory().createParser(new File(fileName))) {
+            if (parser.nextToken() != JsonToken.START_ARRAY) {
+                throw new RuntimeException("JSON file must contain an array at the root");
+            }
+            while (parser.nextToken() == JsonToken.START_OBJECT) {
+                Document doc = objectMapper.readValue(parser, Document.class);
+                //Document doc = new Document();
+                //doc.putAll();
+                counter.incrementAndGet();
+                batch.add(processor.map(doc));
+                if (batch.size() == settings.batchSize) {
+                    final List<Document> batchToInsert = new ArrayList<>(batch);
+                    log.info("Enqueuing " + batch.size() + " rows into collection...");
+                    executor.submit(() -> collection.insertMany(batchToInsert));
+                    batch.clear();
+                }
+            }
+
+            // Process remaining batch
+            if (!batch.isEmpty()) {
+                executor.submit(() -> collection.insertMany(batch));
+            }
+        } finally {
+            executor.shutdown();
+            log.info(counter.get() + " rows enqueued from " + collection + " in " + (System.currentTimeMillis() - startTime) + " ms");
+            try {
+                if (!executor.awaitTermination(settings.timeoutSeconds, TimeUnit.SECONDS)) {
+                    executor.shutdownNow();
+                    log.info(counter.get() + " rows processed in " + (System.currentTimeMillis() - startTime) + " ms");
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                executor.shutdownNow();
+            }
+        }
+    }
+}
diff --git a/...a-db-java-tools/src/main/java/com/datastax/astra/tool/loader/json/JsonLoaderSettings.java b/...a-db-java-tools/src/main/java/com/datastax/astra/tool/loader/json/JsonLoaderSettings.java
@@ -0,0 +1,22 @@
+package com.datastax.astra.tool.loader.json;
+
+import lombok.Builder;
+
+@Builder
+public class JsonLoaderSettings {
+
+    private static final int BATCH_SIZE = 20;
+
+    private static final int THREAD_POOL_SIZE = 5;
+
+    private static final int TIMEOUT = 1800;
+
+    @Builder.Default
+    int batchSize = BATCH_SIZE;;
+
+    @Builder.Default
+    int threadPoolSize = THREAD_POOL_SIZE;;
+
+    @Builder.Default
+    int timeoutSeconds = TIMEOUT;
+}
diff --git a/astra-db-java-tools/src/main/java/com/datastax/astra/tool/loader/json/JsonRecordMapper.java b/astra-db-java-tools/src/main/java/com/datastax/astra/tool/loader/json/JsonRecordMapper.java
@@ -0,0 +1,11 @@
+package com.datastax.astra.tool.loader.json;
+
+import com.datastax.astra.client.collections.definition.documents.Document;
+
+/**
+ * To import a Json containing an arrray of records.
+ */
+public interface JsonRecordMapper {
+
+    Document map(Document jsonRecord);
+}
diff --git a/...stax/astra/tool/loader/pdf/PdfLoader.java → ...stax/astra/tool/loader/pdf/PdfLoader.java b/...stax/astra/tool/loader/pdf/PdfLoader.java → ...stax/astra/tool/loader/pdf/PdfLoader.java
diff --git a/astra-db-java-tools/src/main/java/com/datastax/astra/tool/loader/rag/RagGenericTest.java b/astra-db-java-tools/src/main/java/com/datastax/astra/tool/loader/rag/RagGenericTest.java
@@ -0,0 +1,28 @@
+package com.datastax.astra.tool.loader.rag;
+
+import com.datastax.astra.client.core.query.Projection;
+import com.datastax.astra.client.tables.Table;
+import com.datastax.astra.client.tables.commands.options.TableFindOptions;
+import com.datastax.astra.tool.loader.rag.sources.RagSource;
+
+import java.util.UUID;
+
+public class RagGenericTest {
+
+    public static void main(String[] args) {
+
+        String token = System.getenv("ASTRA_DB_APPLICATION_TOKEN");
+        UUID TEST_TENANT = UUID.fromString("00000000-0000-0000-0000-000000000000");
+
+        //Database db = DataAPIClients.astra(token).getDatabase(TEST_TENANT);
+        //System.out.println(db.getInfo().getName());
+        RagRepository repo = new RagRepository(token, "goodbards");
+        Table<RagSource> tableSources = repo.getTableRagSource(TEST_TENANT);
+
+        TableFindOptions options = new TableFindOptions()
+                .projection(Projection.include("name", "source", "location"));
+        tableSources.find(options).toList().forEach(System.out::println);
+    }
+
+
+}