feat: initialize the project and added qdrant client to push the vectors

.python-version

@@ -0,0 +1 @@
+3.10

main.py

@@ -0,0 +1,104 @@
+import os
+from datetime import datetime
+
+import numpy as np
+from datasets import load_dataset
+from dotenv import load_dotenv
+from qdrant_client import QdrantClient, models
+
+load_dotenv()
+
+repo_id = "mahimairaja/ibm-hls-burn-vectorized"
+ds_from_hub = load_dataset(repo_id)
+
+
+# 2. Initialize Qdrant client and create collection
+client = QdrantClient(
+    url=os.getenv("QDRANT_URL"),
+    api_key=os.getenv("QDRANT_API_KEY"),
+)
+collection_name = "hls_burn_scars_vectorized"
+
+# Recreate the collection with specified vector configurations and payload indexing
+if client.collection_exists(collection_name=collection_name):
+    client.delete_collection(collection_name=collection_name)
+
+client.create_collection(
+    collection_name=collection_name,
+    vectors_config={
+        "dense": models.VectorParams(
+            size=384,
+            distance=models.Distance.COSINE,
+        ),
+        "colbert": models.VectorParams(
+            size=128,
+            distance=models.Distance.COSINE,
+            multivector_config=models.MultiVectorConfig(
+                comparator=models.MultiVectorComparator.MAX_SIM
+            ),
+            hnsw_config=models.HnswConfigDiff(m=0),  # Disable HNSW for reranking
+        ),
+    },
+    # Define payload schema for filtering
+    optimizers_config=models.OptimizersConfigDiff(default_segment_number=2),
+)
+
+# Create payload indexes for filtering
+client.create_payload_index(
+    collection_name=collection_name,
+    field_name="centroid_lat",
+    field_schema=models.Field(field_type=models.PayloadSchemaType.FLOAT),
+)
+client.create_payload_index(
+    collection_name=collection_name,
+    field_name="centroid_lon",
+    field_schema=models.Field(field_type=models.PayloadSchemaType.FLOAT),
+)
+client.create_payload_index(
+    collection_name=collection_name,
+    field_name="acquisition_date",
+    field_schema=models.Field(field_type=models.PayloadSchemaType.DATETIME),
+)
+
+
+# 3. Prepare and ingest data into Qdrant
+def generate_qdrant_points(dataset_split):
+    points = []
+    for i, item in enumerate(dataset_split):
+        # Ensure embeddings are numpy arrays for Qdrant, then convert to list
+        dense_vec = np.array(item["dense_embedding"], dtype=np.float32).tolist()
+        colbert_vec = np.array(item["colbert_embedding"], dtype=np.float32).tolist()
+
+        point = models.PointStruct(
+            id=i,
+            vector={
+                "dense": dense_vec,
+                "colbert": colbert_vec,
+            },
+            payload={
+                "centroid_lat": item["centroid_lat"],
+                "centroid_lon": item["centroid_lon"],
+                "acquisition_date": datetime.strptime(
+                    item["acquisition_date"], "%Y-%m-%d"
+                ),
+            },
+        )
+        points.append(point)
+    return points
+
+
+# Ingest data for each split
+for split_name, dataset_split in ds_from_hub.items():
+    print(f"Ingesting {len(dataset_split)} points from '{split_name}' split...")
+    qdrant_points = generate_qdrant_points(dataset_split)
+
+    client.upsert(collection_name=collection_name, points=qdrant_points, wait=True)
+    print(
+        f"Finished ingesting {len(qdrant_points)} points into Qdrant for '{split_name}' split."
+    )
+
+print("Data ingestion complete for all splits.")
+
+# Verify ingestion by counting points
+count_result = client.count(collection_name=collection_name, exact=True)
+print(f"Total points in Qdrant collection: {count_result.count}")

pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "geo-spatial-chat-qdrant"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "datasets==2.21.0",
+    "python-dotenv>=1.2.1",
+    "qdrant-client>=1.16.1",
+]