Upload 50 files

.env.cloud.sample

@@ -0,0 +1,122 @@
+# ============================================
+# AllyCAT GraphRAG - Cloud Configuration
+# ============================================
+# This configuration uses cloud services for all components
+# Recommended for production and free-tier deployments
+# Docker image size: ~800 MB
+
+# ============================================
+# Pipeline Automation (Docker Only)
+# ============================================
+# Set to 'true' to automatically run the complete pipeline on container startup
+# This will: crawl → process → save to vector DB → process graph → save to graph DB
+# Recommended for cloud deployments (Heroku, AWS, Google Cloud Run)
+AUTO_RUN_PIPELINE=true
+
+# Website to crawl (required if AUTO_RUN_PIPELINE=true)
+WEBSITE_URL=https://your-website.com
+
+# Memory Optimization: Remove pipeline dependencies after completion
+# Saves ~350-500 MB RAM - Highly recommended for 1GB containers
+# Enables deployment on cheaper plans: DigitalOcean $12/mo (1GB) vs $25/mo (2GB)
+CLEANUP_PIPELINE_DEPS=true
+
+# ============================================
+# LLM Configuration - Cloud Mode
+# ============================================
+LLM_RUN_ENV=cloud
+# Choose your preferred cloud LLM (via LiteLLM)
+LLM_MODEL=cerebras/llama3.1-8b
+# Alternative models:
+# LLM_MODEL=gemini/gemini-1.5-flash
+
+# ============================================
+# LLM API Keys (Set at least one)
+# ============================================
+# Cerebras (Fast, free tier available)
+CEREBRAS_API_KEY=your_cerebras_api_key_here
+
+# Google Gemini (Good for graph extraction)
+GEMINI_API_KEY=your_gemini_api_key_here
+
+# Nebius (Alternative provider)
+NEBIUS_API_KEY=your_nebius_api_key_here
+
+# ============================================
+# Vector Database - Zilliz Cloud
+# ============================================
+VECTOR_DB_TYPE=cloud_zilliz
+ZILLIZ_CLUSTER_ENDPOINT=https://your-cluster.zilliz.cloud
+ZILLIZ_TOKEN=your_zilliz_token_here
+
+# ============================================
+# Graph Database - Neo4j Aura Cloud
+# ============================================
+NEO4J_URI=neo4j+s://your-instance.databases.neo4j.io
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD=your_neo4j_password_here
+NEO4J_DATABASE=neo4j
+
+# ============================================
+# Application Settings
+# ============================================
+# Choose app type: flask_graph, chainlit_graph, flask
+APP_TYPE=flask_graph
+
+# ============================================
+# Port Configuration
+# ============================================
+# Flask Applications
+FLASK_VECTOR_PORT=8081      # app_flask.py (vector-only RAG)
+FLASK_GRAPH_PORT=8080       # app_flask_graph.py (GraphRAG - default)
+
+# Chainlit Applications
+CHAINLIT_VECTOR_PORT=8082   # app_chainlit.py
+CHAINLIT_GRAPH_PORT=8083    # app_chainlit_graph.py
+
+# Docker & External Services
+DOCKER_PORT=8080            # External Docker exposed port (host side)
+DOCKER_APP_PORT=8080        # Internal container port (container side, set to match your APP_TYPE)
+OLLAMA_PORT=11434           # Ollama server port (not used in cloud mode)
+
+# Workspace directory
+# For native execution: use relative path 'workspace'
+# For Docker: use absolute path '/allycat/workspace'
+WORKSPACE_DIR=/allycat/workspace
+
+# ============================================
+# Website Crawling Configuration
+# ============================================
+WEBSITE_URL=https://example.com
+CRAWL_MAX_DOWNLOADS=100
+CRAWL_MAX_DEPTH=3
+WAITTIME_BETWEEN_REQUESTS=0.1
+
+# ============================================
+# Embedding Model Configuration
+# ============================================
+EMBEDDING_MODEL=ibm-granite/granite-embedding-30m-english
+EMBEDDING_LENGTH=384
+HF_ENDPOINT=https://hf-mirror.com
+
+# ============================================
+# Chunking Configuration
+# ============================================
+CHUNK_SIZE=512
+CHUNK_OVERLAP=20
+
+# ============================================
+# Graph Extraction Configuration
+# ============================================
+GRAPH_MIN_ENTITIES=5
+GRAPH_MAX_ENTITIES=15
+GRAPH_MIN_RELATIONSHIPS=3
+GRAPH_MAX_RELATIONSHIPS=8
+GRAPH_MIN_CONFIDENCE=0.8
+GRAPH_MAX_CONTENT_CHARS=12000
+GRAPH_SENTENCE_BOUNDARY_RATIO=0.7
+
+# ============================================
+# UI Settings
+# ============================================
+UI_STARTER_PROMPTS=What is this website? | What are upcoming events? | Who are the partners?

.env.hybrid.sample

@@ -0,0 +1,126 @@
+# ============================================
+# AllyCAT GraphRAG - Hybrid Configuration
+# ============================================
+# This configuration uses cloud LLMs with local vector database
+# Recommended for: Privacy-focused deployments with cloud AI benefits
+# Docker image size: ~1.5 GB
+
+# ============================================
+# Pipeline Automation (Docker Only)
+# ============================================
+# Set to 'true' to automatically run the complete pipeline on container startup
+# This will: crawl → process → save to vector DB → process graph → save to graph DB
+AUTO_RUN_PIPELINE=false
+
+# Website to crawl (required if AUTO_RUN_PIPELINE=true)
+WEBSITE_URL=https://your-website.com
+
+# Memory Optimization: Remove pipeline dependencies after completion
+# Saves ~350-500 MB RAM - Useful for hybrid deployments on budget VPS
+CLEANUP_PIPELINE_DEPS=false
+
+# ============================================
+# LLM Configuration - Cloud Mode
+# ============================================
+LLM_RUN_ENV=cloud
+# Choose your preferred cloud LLM (via LiteLLM)
+LLM_MODEL=cerebras/llama3.1-8b
+# Alternative models:
+# LLM_MODEL=gemini/gemini-1.5-flash
+
+# ============================================
+# LLM API Keys (Set at least one)
+# ============================================
+# Cerebras (Fast, free tier available)
+CEREBRAS_API_KEY=your_cerebras_api_key_here
+
+# Google Gemini (Good for graph extraction)
+GEMINI_API_KEY=your_gemini_api_key_here
+
+# Nebius (Alternative provider)
+NEBIUS_API_KEY=your_nebius_api_key_here
+
+# ============================================
+# Vector Database - Local Milvus
+# ============================================
+VECTOR_DB_TYPE=local
+# Local database files stored in workspace
+
+# ============================================
+# Graph Database - Neo4j (Cloud or Local)
+# ============================================
+# Option 1: Neo4j Aura Cloud (Recommended)
+NEO4J_URI=neo4j+s://your-instance.databases.neo4j.io
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD=your_neo4j_password_here
+NEO4J_DATABASE=neo4j
+
+# Option 2: Local Neo4j
+# NEO4J_URI=bolt://localhost:7687
+# NEO4J_USERNAME=neo4j
+# NEO4J_PASSWORD=your_local_password
+# NEO4J_DATABASE=neo4j
+
+# ============================================
+# Application Settings
+# ============================================
+# Choose app type: flask_graph, chainlit_graph, flask, chainlit
+APP_TYPE=flask_graph
+
+# ============================================
+# Port Configuration
+# ============================================
+# Flask Applications
+FLASK_VECTOR_PORT=8081      # app_flask.py (vector-only RAG)
+FLASK_GRAPH_PORT=8080       # app_flask_graph.py (GraphRAG - default)
+
+# Chainlit Applications
+CHAINLIT_VECTOR_PORT=8082   # app_chainlit.py
+CHAINLIT_GRAPH_PORT=8083    # app_chainlit_graph.py
+
+# Docker & External Services
+DOCKER_PORT=8080            # External Docker exposed port (host side)
+DOCKER_APP_PORT=8080        # Internal container port (container side, set to match your APP_TYPE)
+OLLAMA_PORT=11434           # Ollama server port (not used in hybrid mode)
+
+# Workspace directory
+# For native execution: use relative path 'workspace'
+# For Docker: use absolute path '/allycat/workspace'
+WORKSPACE_DIR=/allycat/workspace
+
+# ============================================
+# Website Crawling Configuration
+# ============================================
+WEBSITE_URL=https://example.com
+CRAWL_MAX_DOWNLOADS=100
+CRAWL_MAX_DEPTH=3
+WAITTIME_BETWEEN_REQUESTS=0.1
+
+# ============================================
+# Embedding Model Configuration
+# ============================================
+EMBEDDING_MODEL=ibm-granite/granite-embedding-30m-english
+EMBEDDING_LENGTH=384
+HF_ENDPOINT=https://hf-mirror.com
+
+# ============================================
+# Chunking Configuration
+# ============================================
+CHUNK_SIZE=512
+CHUNK_OVERLAP=20
+
+# ============================================
+# Graph Extraction Configuration
+# ============================================
+GRAPH_MIN_ENTITIES=5
+GRAPH_MAX_ENTITIES=15
+GRAPH_MIN_RELATIONSHIPS=3
+GRAPH_MAX_RELATIONSHIPS=8
+GRAPH_MIN_CONFIDENCE=0.8
+GRAPH_MAX_CONTENT_CHARS=12000
+GRAPH_SENTENCE_BOUNDARY_RATIO=0.7
+
+# ============================================
+# UI Settings
+# ============================================
+UI_STARTER_PROMPTS=What is this website? | What are upcoming events? | Who are the partners?

.env.local.sample

@@ -0,0 +1,125 @@
+# ============================================
+# AllyCAT GraphRAG - Local Configuration
+# ============================================
+# This configuration uses local services for all components
+# Recommended for development, testing, and offline deployments
+# Docker image size: ~4+ GB (includes Ollama)
+
+# ============================================
+# Pipeline Automation (Docker Only)
+# ============================================
+# Set to 'true' to automatically run the complete pipeline on container startup
+# This will: crawl → process → save to vector DB → process graph → save to graph DB
+# For local development, typically set to 'false' to run steps manually
+AUTO_RUN_PIPELINE=false
+
+# Website to crawl (required if AUTO_RUN_PIPELINE=true)
+WEBSITE_URL=https://your-website.com
+
+# Memory Optimization: Remove pipeline dependencies after completion
+# Saves ~350-500 MB RAM (less critical for local development)
+# Set to true if running in resource-constrained environments
+CLEANUP_PIPELINE_DEPS=false
+
+# ============================================
+# LLM Configuration - Local Ollama
+# ============================================
+LLM_RUN_ENV=local_ollama
+LLM_MODEL=ollama/gemma3:1b
+# Model to download and use
+OLLAMA_MODEL=gemma3:1b
+# Alternative local models:
+# OLLAMA_MODEL=qwen2.5:1.5b
+# OLLAMA_MODEL=llama3.2:1b
+
+# ============================================
+# Vector Database - Local Milvus
+# ============================================
+VECTOR_DB_TYPE=local
+# Local database files stored in workspace
+
+# ============================================
+# Graph Database - Local or Cloud Neo4j
+# ============================================
+# Option 1: Local Neo4j (requires separate Neo4j installation)
+NEO4J_URI=bolt://localhost:7687
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD=your_local_password
+NEO4J_DATABASE=neo4j
+
+# Option 2: Or use Neo4j Aura Cloud even in local mode
+# NEO4J_URI=neo4j+s://your-instance.databases.neo4j.io
+# NEO4J_USERNAME=neo4j
+# NEO4J_PASSWORD=your_neo4j_password_here
+# NEO4J_DATABASE=neo4j
+
+# ============================================
+# Graph Extraction LLM (Cloud API recommended)
+# ============================================
+# Even in local mode, graph extraction benefits from cloud LLMs
+# Set at least one for graph building:
+GEMINI_API_KEY=your_gemini_api_key_here
+CEREBRAS_API_KEY=your_cerebras_api_key_here
+
+# ============================================
+# Application Settings
+# ============================================
+APP_TYPE=flask_graph
+
+# ============================================
+# Port Configuration
+# ============================================
+# Flask Applications
+FLASK_VECTOR_PORT=8081      # app_flask.py (vector-only RAG)
+FLASK_GRAPH_PORT=8080       # app_flask_graph.py (GraphRAG - default)
+
+# Chainlit Applications
+CHAINLIT_VECTOR_PORT=8082   # app_chainlit.py
+CHAINLIT_GRAPH_PORT=8083    # app_chainlit_graph.py
+
+# Docker & External Services
+DOCKER_PORT=8080            # External Docker exposed port (host side)
+DOCKER_APP_PORT=8080        # Internal container port (container side, set to match your APP_TYPE)
+OLLAMA_PORT=11434           # Ollama server port (for local LLM)
+
+# Workspace directory
+# For native execution: use relative path 'workspace'
+# For Docker: use absolute path '/allycat/workspace'
+WORKSPACE_DIR=/allycat/workspace
+
+# ============================================
+# Website Crawling Configuration
+# ============================================
+WEBSITE_URL=https://example.com
+CRAWL_MAX_DOWNLOADS=100
+CRAWL_MAX_DEPTH=3
+WAITTIME_BETWEEN_REQUESTS=0.1
+
+# ============================================
+# Embedding Model Configuration
+# ============================================
+EMBEDDING_MODEL=ibm-granite/granite-embedding-30m-english
+EMBEDDING_LENGTH=384
+HF_ENDPOINT=https://hf-mirror.com
+
+# ============================================
+# Chunking Configuration
+# ============================================
+CHUNK_SIZE=512
+CHUNK_OVERLAP=20
+
+# ============================================
+# Graph Extraction Configuration
+# ============================================
+GRAPH_MIN_ENTITIES=5
+GRAPH_MAX_ENTITIES=15
+GRAPH_MIN_RELATIONSHIPS=3
+GRAPH_MAX_RELATIONSHIPS=8
+GRAPH_MIN_CONFIDENCE=0.8
+GRAPH_MAX_CONTENT_CHARS=12000
+GRAPH_SENTENCE_BOUNDARY_RATIO=0.7
+
+# ============================================
+# UI Settings
+# ============================================
+UI_STARTER_PROMPTS=What is this website? | What are upcoming events? | Who are the partners?

.gitignore

@@ -0,0 +1,26 @@
+workspace/
+*.out*
+.directory
+venv*
+.vscode
+tmp
+
+# Ignore actual .env file but allow sample files
+.env
+!.env.*.sample
+!env.sample.txt
+
+*.db
+*.db.lock
+
+## profiling outputs
+*.speed
+
+__pycache__
+
+chainlit.md
+
+node_modules/
+
+logs/
+logs/*

.python-version

@@ -0,0 +1 @@
+3.11

1_crawl_site.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+
+import argparse
+import shutil
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+import time
+import logging
+import os
+import re
+import mimetypes
+from my_config import MY_CONFIG
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+class WebScraper:
+    def __init__(self, url, max_downloads, depth):
+        self.url = url
+        self.max_downloads = max_downloads
+        self.depth = depth
+        self.visited_urls = set()
+        self.downloaded_base_urls = set()  # Track base URLs without fragments
+        self.downloaded_count = 0
+        
+    def scrape_page(self, url, current_depth=0):
+        try:
+            # For downloading, we need to remove fragment since HTTP requests ignore them
+            parsed_url = urlparse(url)
+            download_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
+            if parsed_url.query:
+                download_url += f"?{parsed_url.query}"
+            
+            # Check if we've already downloaded this base URL content
+            if download_url in self.downloaded_base_urls:
+                # If we have a fragment and haven't visited this exact URL, save with fragment name
+                if parsed_url.fragment and url not in self.visited_urls:
+                    # Get the cached response if we have one, otherwise make a request
+                    response = requests.get(download_url, timeout=10)
+                    response.raise_for_status()
+                    
+                    filename = self.url_to_filename(url, response)
+                    filepath = os.path.join(MY_CONFIG.CRAWL_DIR, filename)
+                    
+                    # Handle binary files vs text files based on mime type
+                    mime_type = response.headers.get('Content-Type', '').lower()
+                    is_text = mime_type.startswith('text/') or 'html' in mime_type or 'xml' in mime_type
+                    
+                    if is_text:
+                        with open(filepath, 'w', encoding='utf-8') as f:
+                            f.write(response.text)
+                    else:
+                        with open(filepath, 'wb') as f:
+                            f.write(response.content)
+                    
+                    self.downloaded_count += 1
+                    logger.info(f"Saved {filepath} with fragment ({self.downloaded_count}/{self.max_downloads})")
+                    
+                    return []  # Don't re-parse links from same content
+                else:
+                    logger.info(f"Skipping already downloaded URL: {download_url}")
+                    return []
+            
+            response = requests.get(download_url, timeout=10)
+            response.raise_for_status()
+            
+            # Track that we've downloaded this base URL
+            self.downloaded_base_urls.add(download_url)
+            
+            # Save file using original URL (with fragment) for unique filename
+            filename = self.url_to_filename(url, response)
+            filepath = os.path.join(MY_CONFIG.CRAWL_DIR, filename)
+            
+            # Handle binary files vs text files based on mime type
+            mime_type = response.headers.get('Content-Type', '').lower()
+            is_text = mime_type.startswith('text/') or 'html' in mime_type or 'xml' in mime_type
+            
+            if is_text:
+                with open(filepath, 'w', encoding='utf-8') as f:
+                    f.write(response.text)
+            else:
+                with open(filepath, 'wb') as f:
+                    f.write(response.content)
+            
+            self.downloaded_count += 1
+            logger.info(f"Saved {filepath} ({self.downloaded_count}/{self.max_downloads})")
+            
+            # Parse for links if not at max depth
+            links = []
+            if current_depth < self.depth:
+                soup = BeautifulSoup(response.content, 'html.parser')
+                base_domain = urlparse(self.url).netloc
+                for link in soup.find_all('a', href=True):
+                    full_url = urljoin(url, link.get('href'))
+                    if urlparse(full_url).netloc == base_domain:
+                        links.append(full_url)
+            
+            return links
+            
+        except Exception as e:
+            logger.error(f"Error scraping {url}: {str(e)}")
+            return []
+    
+    def url_to_filename(self, url, response):
+        # Keep domain and path, strip protocol, use __ for directory separators
+        parsed = urlparse(url)
+        domain = parsed.netloc
+        path = parsed.path
+        fragment = parsed.fragment
+        
+        if not path or path == '/':
+            filename = f"{domain}__index"
+        else:
+            filename = f"{domain}{path.replace('/', '__')}"
+        
+        # Add fragment (anchor) to filename if present
+        if fragment:
+            filename = f"{filename}__{fragment}"
+        
+        filename = re.sub(r'[^\w\-_.]', '_', filename)
+        
+        mime_type = response.headers.get('Content-Type')
+        if mime_type:
+            inferred_extension = mimetypes.guess_extension(mime_type.split(';')[0].strip())
+        else:            
+            inferred_extension = '.html'
+
+        current_ext = os.path.splitext(filename)[1]
+        ext = os.path.splitext(filename)[1]
+        # print ('--- filename:', filename)  # Debugging line
+        # print ('--- mimetype:', mime_type)  # Debugging line
+        # print ('--- inferred_extension', inferred_extension)  # Debugging line
+        # print ('--- current_ext:', current_ext)  # Debugging line
+
+        # Only append .html if no extension exists
+        if not filename.endswith(inferred_extension):
+             filename = f"{filename}.html"
+
+        # print ('--- returning filename:', filename)  # Debugging line
+        return filename
+        
+    
+    def scrape(self):
+        shutil.rmtree(MY_CONFIG.CRAWL_DIR, ignore_errors=True)
+        os.makedirs(MY_CONFIG.CRAWL_DIR, exist_ok=True)
+        logger.info(f"✅ Cleared  crawl directory: {MY_CONFIG.CRAWL_DIR}")
+
+        logger.info(f"⚙ Starting scrape of {self.url}, max downloads: {self.max_downloads}, depth: {self.depth}")
+
+        
+        urls_to_visit = [(self.url, 0)]  # (url, depth)
+        
+        while urls_to_visit and self.downloaded_count < self.max_downloads:
+            current_url, current_depth = urls_to_visit.pop(0)
+            
+            if current_url in self.visited_urls:
+                continue
+                
+            self.visited_urls.add(current_url)
+            
+            links = self.scrape_page(current_url, current_depth)
+            
+            # Add new URLs if not at max depth
+            if current_depth < self.depth:
+                for link in links:
+                    if link not in self.visited_urls:
+                        urls_to_visit.append((link, current_depth + 1))
+            
+            time.sleep(MY_CONFIG.WAITTIME_BETWEEN_REQUESTS)
+    
+
+def main():
+    parser = argparse.ArgumentParser(description="Web scraper")
+    parser.add_argument("--url", type=str, default=MY_CONFIG.WEBSITE_URL, help=f"URL to scrape (default: {MY_CONFIG.WEBSITE_URL})")
+    parser.add_argument("--max-downloads", type=int, default=MY_CONFIG.CRAWL_MAX_DOWNLOADS, help=f"Maximum number of files to download (default: {MY_CONFIG.CRAWL_MAX_DOWNLOADS})")
+    parser.add_argument("--depth", type=int, default=MY_CONFIG.CRAWL_MAX_DEPTH, help=f"Maximum depth to crawl (default: {MY_CONFIG.CRAWL_MAX_DEPTH})")
+
+    args = parser.parse_args()
+    
+    scraper = WebScraper(args.url, args.max_downloads, args.depth)
+    scraper.scrape()
+    
+    logger.info(f"✅ Scraping completed. Downloaded {scraper.downloaded_count}  files to '{MY_CONFIG.CRAWL_DIR}' directory.")
+
+if __name__ == "__main__":
+    main()

2_process_files.ipynb

@@ -0,0 +1,135 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Processing HTML Files\n",
+    "\n",
+    "We will be using **docling**\n",
+    "\n",
+    "References\n",
+    "- [docling](https://github.com/DS4SD/docling)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-1: Data\n",
+    "\n",
+    "We will process data that is downloaded using [1_crawl_site.ipynb](1_crawl_site.ipynb).\n",
+    "\n",
+    "We have a couple of crawled HTML files in  `input` directory. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-2: Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## All config is defined here\n",
+    "from my_config import MY_CONFIG"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Cleared  processed data directory :  workspace/processed\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os, sys\n",
+    "import shutil\n",
+    "\n",
+    "shutil.rmtree(MY_CONFIG.PROCESSED_DATA_DIR, ignore_errors=True)\n",
+    "shutil.os.makedirs(MY_CONFIG.PROCESSED_DATA_DIR, exist_ok=True)\n",
+    "print (f\"✅ Cleared  processed data directory :  {MY_CONFIG.PROCESSED_DATA_DIR}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-3: Convet FILES --> MD\n",
+    "\n",
+    "Process HTML documents and extract the text in markdown format"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time \n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "from docling.document_converter import DocumentConverter\n",
+    "\n",
+    "converter = DocumentConverter(format_options={\"preserve_links\": True})\n",
+    "\n",
+    "input_path = Path(MY_CONFIG.CRAWL_DIR)\n",
+    "input_files = list(input_path.glob('*.html')) + list(input_path.glob('*.htm')) + list(input_path.glob('*.pdf'))\n",
+    "print (f\"Found {len(input_files)} files to convert\")\n",
+    "\n",
+    "files_processed = 0\n",
+    "errors = 0\n",
+    "for input_file in input_files:\n",
+    "    try:\n",
+    "        result = converter.convert(input_file)\n",
+    "        markdown_content = result.document.export_to_markdown()\n",
+    "        \n",
+    "        md_file_name = os.path.join(MY_CONFIG.PROCESSED_DATA_DIR, f\"{input_file.stem}.md\")\n",
+    "        with open(md_file_name, \"w\", encoding=\"utf-8\") as md_file:\n",
+    "            md_file.write(markdown_content)\n",
+    "            \n",
+    "        print (f\"Converted '{input_file}' --> '{md_file_name}'\")\n",
+    "        files_processed += 1\n",
+    "    except Exception as e:\n",
+    "        errors += 1\n",
+    "        print (f\"Error processing {input_file}: {e}\")\n",
+    "\n",
+    "print (f\"✅ Processed {files_processed} files.  Errors: {errors}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "allycat-1",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

2_process_files.py

@@ -0,0 +1,141 @@
+import os, sys
+import shutil
+from pathlib import Path
+from docling.document_converter import DocumentConverter
+import html2text
+import logging
+import hashlib
+from my_config import MY_CONFIG
+
+logging.basicConfig(level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+def cleanup_duplicate_markdown_files(processed_dir):
+    """
+    Remove duplicate markdown files based on content hash.
+    Keeps the first file encountered for each unique content.
+    """
+    processed_path = Path(processed_dir)
+    md_files = list(processed_path.glob('*.md'))
+    
+    if not md_files:
+        logger.info("No markdown files found for deduplication")
+        return 0
+    
+    content_hashes = {}
+    duplicates_removed = 0
+    
+    for md_file in md_files:
+        try:
+            with open(md_file, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
+            
+            if content_hash in content_hashes:
+                os.remove(md_file)
+                duplicates_removed += 1
+                logger.info(f"Removed duplicate: {md_file} (same content as {content_hashes[content_hash]})")
+            else:
+                content_hashes[content_hash] = md_file
+                
+        except Exception as e:
+            logger.warning(f"Error processing {md_file} for deduplication: {e}")
+    
+    logger.info(f"✅ Deduplication complete. Removed {duplicates_removed} duplicate files")
+    return duplicates_removed
+## --- end of cleanup_duplicate_markdown_files ---
+
+def process_files(crawl_dir, processed_dir):
+    """
+    Process all files in the crawl directory and convert them to markdown.
+    Uses html2text for HTML/HTM files and docling for PDFs and other documents.
+    
+    Args:
+        crawl_dir (str): Directory containing files to process
+        processed_dir (str): Directory to save processed markdown files
+    """
+
+    input_path = Path(crawl_dir)
+    input_files = list(input_path.glob('*')) 
+    logger.info (f"Found {len(input_files)} files to process in {input_path}")
+
+    shutil.rmtree(processed_dir, ignore_errors=True)
+    shutil.os.makedirs(processed_dir, exist_ok=True)
+    logger.info (f"✅ Cleared  processed data directory :  {processed_dir}")
+    
+    # Initialize converters
+    docling_converter = DocumentConverter(format_options={"preserve_links": True})
+    html_converter = html2text.HTML2Text()
+    html_converter.ignore_links = False
+    html_converter.ignore_images = False
+    
+    files_processed = 0
+    errors = 0
+    file_type_stats = {}
+    
+    for input_file in input_files:
+        file_ext = input_file.suffix.lower()
+        markdown_content = None
+        
+        try:
+            # Process HTML/HTM files with html2text
+            if file_ext in ['.html', '.htm']:
+                with open(input_file, 'r', encoding='utf-8', errors='ignore') as f:
+                    html_content = f.read()
+                markdown_content = html_converter.handle(html_content)
+                logger.debug(f"Converted HTML '{input_file}' with html2text")
+            
+            # Process TXT files directly
+            elif file_ext == '.txt':
+                with open(input_file, 'r', encoding='utf-8', errors='ignore') as f:
+                    markdown_content = f.read()
+                logger.debug(f"Processed TXT '{input_file}' directly")
+            
+            # Process PDF and other documents with docling
+            else:
+                result = docling_converter.convert(input_file)
+                markdown_content = result.document.export_to_markdown()
+                logger.debug(f"Converted '{input_file}' with docling")
+            
+            # Save markdown file
+            if markdown_content:
+                md_file_name = os.path.join(processed_dir, f"{input_file.stem}.md")
+                with open(md_file_name, "w", encoding="utf-8") as md_file:
+                    md_file.write(markdown_content)
+                
+                files_processed += 1
+                file_type_stats[file_ext] = file_type_stats.get(file_ext, 0) + 1
+            
+        except Exception as e:
+            errors += 1
+            logger.warning(f"Error processing {input_file}: {e}")
+
+    logger.info (f"✅ Processed {files_processed} files.  Errors: {errors}")
+    
+    # Print file type statistics in compact dictionary format
+    if file_type_stats:
+        logger.info(f"📊 File type statistics: {dict(sorted(file_type_stats.items()))}")
+    
+    return files_processed, errors, file_type_stats
+## --- end of process_files ---
+
+def main():
+    """
+    Main function to run the file processing pipeline.
+    """
+    logger.info("🚀 Starting file processing pipeline")
+    
+    try:
+        files_processed, errors, file_type_stats = process_files(MY_CONFIG.CRAWL_DIR, MY_CONFIG.PROCESSED_DATA_DIR)
+        duplicates_removed = cleanup_duplicate_markdown_files(MY_CONFIG.PROCESSED_DATA_DIR)
+        logger.info(f"✅ Final summary: {files_processed} files processed, {errors} errors, {duplicates_removed} duplicates removed")
+        logger.info("✅ File processing pipeline completed successfully")
+        return 0
+    except Exception as e:
+        logger.error(f"❌ File processing pipeline failed: {e}")
+        return 1
+
+if __name__ == "__main__":
+    sys.exit(main())

2b_process_graph_phase1.py

@@ -0,0 +1,881 @@
+"""
+GraphRAG Phase 1: LLM-based Entity and Relationship Extraction
+Builds initial knowledge graph from markdown files using LLMs (Cerebras or Gemini)
+"""
+
+import json
+import logging
+import os
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Dict, List
+from datetime import datetime
+import orjson
+from json_repair import repair_json
+import google.generativeai as genai
+import openai
+from my_config import MY_CONFIG
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+class GraphBuilder:
+    
+    def __init__(self, llm_provider="cerebras"):
+        self.llm_provider = llm_provider.lower()
+        
+        # Global entity registry for deduplication across files
+        self.global_entity_registry = {}
+        
+        # Initialize graph data structure
+        self.graph_data = {"nodes": [], "relationships": []}
+        self.processed_files = 0
+        
+        # Initialize LLM API based on provider
+        if self.llm_provider == "cerebras":
+            if not MY_CONFIG.CEREBRAS_API_KEY:
+                raise ValueError("CEREBRAS_API_KEY environment variable not set. Get free key at: https://cloud.cerebras.ai/")
+            
+            # Configure Cerebras client
+            self.cerebras_client = openai.OpenAI(
+                api_key=MY_CONFIG.CEREBRAS_API_KEY,
+                base_url="https://api.cerebras.ai/v1"
+            )
+            self.model_name = "llama-4-scout-17b-16e-instruct"  
+            logger.info("🚀 Using Cerebras API")
+            
+        elif self.llm_provider == "gemini":
+            if not MY_CONFIG.GEMINI_API_KEY:
+                raise ValueError("GEMINI_API_KEY environment variable not set. Get free key at: https://aistudio.google.com/")
+            
+            # Configure Gemini with FREE tier
+            genai.configure(api_key=MY_CONFIG.GEMINI_API_KEY)
+            self.model_name = "gemini-1.5-flash" 
+            self.gemini_model = genai.GenerativeModel(self.model_name)
+            logger.info("🆓 Using Google Gemini API,)")
+            
+        else:
+            valid_providers = ["cerebras", "gemini"]
+            raise ValueError(f"Invalid provider '{llm_provider}'. Choose from: {valid_providers}")
+
+        # Configure extraction parameters
+        self.min_entities = int(os.getenv("GRAPH_MIN_ENTITIES", "5"))
+        self.max_entities = int(os.getenv("GRAPH_MAX_ENTITIES", "15"))
+        self.min_relationships = int(os.getenv("GRAPH_MIN_RELATIONSHIPS", "3"))
+        self.max_relationships = int(os.getenv("GRAPH_MAX_RELATIONSHIPS", "8"))
+        self.min_confidence = float(os.getenv("GRAPH_MIN_CONFIDENCE", "0.8"))
+        self.max_content_chars = int(os.getenv("GRAPH_MAX_CONTENT_CHARS", "12000"))
+        self.sentence_boundary_ratio = float(os.getenv("GRAPH_SENTENCE_BOUNDARY_RATIO", "0.7"))
+        
+        logger.info(f"✅ Initialized {self.llm_provider.upper()} provider with model: {self.model_name}")
+        logger.info(f"Extraction config: {self.min_entities}-{self.max_entities} entities, {self.min_relationships}-{self.max_relationships} relationships, min confidence: {self.min_confidence}")
+        logger.info(f"Content processing: {self.max_content_chars} chars per chunk with overlap for FULL analysis")
+
+    # STEP 0: Clean Graph Data Folder
+    def clean_graph_folder(self, graph_dir: str = None):
+        if graph_dir is None:
+            graph_dir = "workspace/graph_data"
+        try:
+            graph_path = Path(graph_dir)
+            if graph_path.exists():
+                # Remove all files in the directory
+                for file_path in graph_path.glob("*"):
+                    if file_path.is_file():
+                        file_path.unlink()
+                        logger.debug(f"Removed: {file_path.name}")
+                logger.info(f"Cleaned graph folder: {graph_dir}")
+            else:
+                # Create directory if it doesn't exist
+                graph_path.mkdir(parents=True, exist_ok=True)
+                logger.info(f"Created graph folder: {graph_dir}")
+        except Exception as e:
+            logger.warning(f"Failed to clean graph folder: {e}")
+
+    # STEP 1: Content Preprocessing and Chunking
+    def _preprocess_content(self, text: str, max_chars: int = None) -> str:
+        # Remove excessive whitespace but keep full content
+        text = ' '.join(text.split())
+        return text.strip()
+    
+    def _chunk_content(self, text: str, chunk_size: int = None, overlap: int = 200) -> List[str]:
+        if chunk_size is None:
+            chunk_size = self.max_content_chars
+            
+        # If content fits in one chunk, return as-is
+        if len(text) <= chunk_size:
+            return [text]
+        
+        chunks = []
+        start = 0
+        
+        while start < len(text):
+            # Calculate end position
+            end = start + chunk_size
+            
+            if end >= len(text):
+                # Last chunk
+                chunks.append(text[start:])
+                break
+            
+            # Try to find good break point (sentence boundary)
+            chunk_text = text[start:end]
+            last_period = chunk_text.rfind('.')
+            last_newline = chunk_text.rfind('\n')
+            
+            # Use best break point
+            break_point = max(last_period, last_newline)
+            if break_point > chunk_size * 0.7:  # Good break point
+                actual_end = start + break_point + 1
+                chunks.append(text[start:actual_end])
+                start = actual_end - overlap  # Overlap for context
+            else:
+                # No good break point, use hard split
+                chunks.append(text[start:end])
+                start = end - overlap
+                
+        return chunks
+
+    # STEP 2: LLM Prompt Generation
+    def get_entity_extraction_prompt(self) -> str:
+        return f"""You are a specialized knowledge graph extraction assistant. Your task is to analyze content and extract entities and relationships to build comprehensive knowledge graphs.
+
+DYNAMIC EXTRACTION REQUIREMENTS:
+- Extract {self.min_entities}-{self.max_entities} most important entities from the content
+- Create {self.min_relationships}-{self.max_relationships} meaningful relationships between entities
+- Confidence threshold: {self.min_confidence} (only include high-confidence extractions)
+- Focus on extracting diverse entity types relevant to the content domain
+
+CONSTITUTIONAL AI PRINCIPLES:
+1. Content-Adaptive: Determine entity types based on content analysis, not predefined categories
+2. Relationship-Rich: Focus on meaningful semantic relationships between entities
+3. Context-Aware: Consider document context and domain when extracting entities
+4. Quality-First: Prioritize extraction quality over quantity
+
+ENTITY EXTRACTION GUIDELINES:
+- Identify the most important concepts, terms, people, places, organizations, technologies, events
+- Extract entities that would be valuable for knowledge graph queries
+- Include both explicit entities (directly mentioned) and implicit entities (strongly implied)
+- Assign appropriate types based on semantic analysis of the entity's role in the content
+
+RELATIONSHIP EXTRACTION GUIDELINES:
+- Create relationships that capture semantic meaning, not just co-occurrence
+- Use descriptive relationship types that express the nature of the connection
+- Include hierarchical, associative, and causal relationships where appropriate
+- Ensure relationships are bidirectionally meaningful and contextually accurate
+
+OUTPUT FORMAT (strict JSON):
+{{
+    "entities": [
+        {{
+            "text": "Entity Name",
+            "type": "DynamicType",
+            "content": "Comprehensive description of the entity",
+            "confidence": 0.95
+        }}
+    ],
+    "relationships": [
+        {{
+            "startNode": "Entity Name 1",
+            "endNode": "Entity Name 2",
+            "type": "DESCRIPTIVE_RELATIONSHIP_TYPE",
+            "description": "Clear description of the relationship",
+            "evidence": "Direct evidence from text supporting this relationship",
+            "confidence": 0.90
+        }}
+    ]
+}}
+
+IMPORTANT: Respond with ONLY the JSON object. No explanations, no markdown formatting, no code blocks."""
+
+    # STEP 3: LLM Inference Methods  
+    def _cerebras_inference(self, system_prompt: str, user_prompt: str) -> str:
+        try:
+            # Cerebras uses OpenAI-compatible chat format
+            response = self.cerebras_client.chat.completions.create(
+                model=self.model_name,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt}
+                ],
+                temperature=0.1,
+                max_tokens=2000
+            )
+            
+            # Check for empty response 
+            if not response or not response.choices or not response.choices[0].message.content:
+                raise ValueError("Empty response from Cerebras")
+            
+            return response.choices[0].message.content.strip()
+            
+        except Exception as e:
+            # Check for quota/rate limit exceeded errors
+            error_str = str(e).lower()
+            if "429" in str(e) and "quota" in error_str:
+                logger.error(f"🚫 QUOTA EXCEEDED: Cerebras API rate/quota limit reached - {e}")
+                raise Exception("QUOTA_EXCEEDED") from e
+            else:
+                logger.error(f"Error with Cerebras inference: {e}")
+                raise e
+    
+    def _gemini_inference(self, system_prompt: str, user_prompt: str) -> str:
+        try:
+            combined_prompt = f"{system_prompt}\n\n{user_prompt}"
+            response = self.gemini_model.generate_content(combined_prompt)    
+            if not response or not response.text:
+                raise ValueError("Empty response from Gemini")
+                
+            return response.text.strip()
+            
+        except Exception as e:
+            # Check for quota exceeded error
+            if "429" in str(e) and "quota" in str(e).lower():
+                logger.error(f"🚫 QUOTA EXCEEDED: Gemini API daily limit reached - {e}")
+                raise Exception("QUOTA_EXCEEDED") from e
+            else:
+                logger.error(f"Error with Gemini inference: {e}")
+                raise e
+    
+    # STEP 4: JSON Parsing Pipeline
+    def _smart_json_parse(self, json_text: str) -> Dict[str, Any]:
+        
+        cleaned_text = json_text.strip()
+        
+        # Step 1: orjson
+        try:
+            result = orjson.loads(cleaned_text.encode('utf-8'))
+            logger.debug("✅ Step 1: orjson succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 1: orjson failed - {e}")
+        
+        # Step 2: json-repair
+        try:
+            repaired = repair_json(cleaned_text)
+            result = orjson.loads(repaired.encode('utf-8'))
+            logger.debug("✅ Step 2: json-repair + orjson succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 2: json-repair failed - {e}")
+        
+        # Step 3: standard json
+        try:
+            result = json.loads(cleaned_text)
+            logger.debug("✅ Step 3: standard json succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 3: standard json failed - {e}")
+        
+        # Step 4: json-repair + standard json
+        try:
+            repaired = repair_json(cleaned_text)
+            result = json.loads(repaired)
+            logger.debug("✅ Step 4: json-repair + standard json succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 4: json-repair + standard json failed - {e}")
+        
+        # Step 5: All failed - this will trigger save failed txt files
+        raise ValueError("All 4 JSON parsing steps failed")
+
+    # STEP 5: Response Parsing and Validation
+    def _parse_llm_extraction_response(self, llm_response: str, file_name: str) -> Dict[str, Any]:
+        
+        # Clean up response first
+        cleaned_response = llm_response.strip()
+        
+        # Remove markdown formatting
+        if "```json" in cleaned_response:
+            parts = cleaned_response.split("```json")
+            if len(parts) > 1:
+                json_part = parts[1].split("```")[0].strip()
+                cleaned_response = json_part
+        elif "```" in cleaned_response:
+            parts = cleaned_response.split("```")
+            if len(parts) >= 3:
+                cleaned_response = parts[1].strip()
+        
+        # Use the 5-step JSON parsing pipeline
+        try:
+            extraction_data = self._smart_json_parse(cleaned_response)
+            
+            # Validate complete format
+            if self._validate_complete_format(extraction_data):
+                return extraction_data
+            else:
+                self._save_failed_response(cleaned_response, file_name, "Format validation failed", "Missing required fields or empty values")
+                return None
+        except Exception as e:
+            logger.error(f"❌ All JSON parsing steps failed for file {file_name}: {str(e)}")
+            self._save_failed_response(cleaned_response, file_name, "All parsing steps failed", str(e))
+            return None
+    
+    # STEP 6: Format Validation
+    def _validate_complete_format(self, extraction_data: Dict[str, Any]) -> bool:
+
+        if not isinstance(extraction_data, dict):
+            return False
+
+        if "entities" not in extraction_data or "relationships" not in extraction_data:
+            return False
+
+        entities = extraction_data.get("entities", [])
+        relationships = extraction_data.get("relationships", [])
+        if not isinstance(entities, list) or len(entities) == 0:
+            return False
+        for entity in entities:
+            if not isinstance(entity, dict):
+                return False
+            
+            required_fields = ["text", "type", "content", "confidence"]
+            for field in required_fields:
+                if field not in entity:
+                    return False
+                value = entity[field]
+                if value is None or value == "" or (isinstance(value, str) and not value.strip()):
+                    return False
+                
+            if not isinstance(entity["confidence"], (int, float)) or entity["confidence"] <= 0:
+                return False
+        
+        if isinstance(relationships, list):
+            for rel in relationships:
+                if not isinstance(rel, dict):
+                    return False
+                
+                required_fields = ["startNode", "endNode", "type", "description", "evidence", "confidence"]
+                for field in required_fields:
+                    if field not in rel:
+                        return False
+                    value = rel[field]
+                    if value is None or value == "" or (isinstance(value, str) and not value.strip()):
+                        return False
+                
+                if not isinstance(rel["confidence"], (int, float)) or rel["confidence"] <= 0:
+                    return False
+        
+        return True
+    
+    # STEP 7: Error Handling and Failed Response Logging
+    def _save_failed_response(self, llm_response: str, file_name: str, _json_error: str, _repair_error: str):
+        try:
+            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            output_dir = Path("workspace/graph_data")
+            output_dir.mkdir(parents=True, exist_ok=True)
+    
+            with open(output_dir / "failed_responses.txt", 'a', encoding='utf-8') as f:
+                f.write(f"# Failed response from file: {file_name} at {timestamp}\n")
+                f.write(llm_response)
+                f.write("\n---\n") 
+                f.flush()  
+                
+        except Exception as save_error:
+            logger.error(f"Failed to save failed response from {file_name}: {save_error}")
+            
+    # STEP 8: Main Entity Extraction
+    def extract_entities_with_llm(self, content: str, file_name: str) -> Dict[str, Any]:
+        # Preprocess content
+        processed_content = self._preprocess_content(content)
+        
+        # Split into chunks 
+        chunks = self._chunk_content(processed_content)
+        
+        logger.info(f"📄 Processing {file_name}: {len(processed_content)} chars in {len(chunks)} chunk(s)")
+        
+        # Collect all entities and relationships from chunks
+        all_entities = []
+        all_relationships = []
+        
+        for chunk_idx, chunk in enumerate(chunks):
+            logger.info(f"🔄 Processing chunk {chunk_idx + 1}/{len(chunks)} for {file_name}")
+            
+            # Simple retry mechanism for empty content - just send to LLM again
+            max_retries = 3
+            for attempt in range(max_retries):
+                # Get the optimized prompt for entity extraction based on provider
+                system_prompt = self.get_entity_extraction_prompt()
+                
+                # Create user prompt with chunk content
+                chunk_info = f" (chunk {chunk_idx + 1}/{len(chunks)})" if len(chunks) > 1 else ""
+                user_prompt = f"""
+                Analyze the following content from file "{file_name}"{chunk_info}:
+                
+                ```
+                {chunk}
+                ```
+                
+                Extract all relevant entities, concepts, and their relationships from this content.
+                """
+                
+                # Call appropriate LLM API
+                try:
+                    if self.llm_provider == "gemini":
+                        llm_response = self._gemini_inference(system_prompt, user_prompt)
+                    elif self.llm_provider == "cerebras":
+                        llm_response = self._cerebras_inference(system_prompt, user_prompt)
+                    else:
+                        raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
+                except Exception as e:
+                    if "QUOTA_EXCEEDED" in str(e):
+                        logger.error(f"🚫 QUOTA EXCEEDED on file {file_name}, chunk {chunk_idx + 1} - stopping processing")
+                        # Return partial results if we have any
+                        return {
+                            "entities": all_entities,
+                            "relationships": all_relationships,
+                            "file": file_name,
+                            "structure": {"section": "partial_quota_exceeded"},
+                            "chunks_processed": chunk_idx,
+                            "total_content_length": len(processed_content),
+                            "quota_exceeded": True
+                        }
+                    else:
+                        raise e
+                    
+                # Parse the JSON response
+                result = self._parse_llm_extraction_response(llm_response, f"{file_name}_chunk_{chunk_idx}")
+                if result is not None or attempt == max_retries - 1:
+                    if result is None:
+                        logger.warning(f"❌ Chunk {chunk_idx + 1} of {file_name} failed all validation attempts, skipping")
+                        break
+
+                    # Chunk results to collections
+                    chunk_entities = result.get("entities", [])
+                    chunk_relationships = result.get("relationships", [])
+                    
+                    # Add chunk identifier to entities for deduplication
+                    for entity in chunk_entities:
+                        entity["chunk_id"] = chunk_idx
+                        entity["source_chunk"] = f"chunk_{chunk_idx}"
+                    
+                    # Add chunk identifier to relationships
+                    for rel in chunk_relationships:
+                        rel["chunk_id"] = chunk_idx
+                        rel["source_chunk"] = f"chunk_{chunk_idx}"
+                    
+                    all_entities.extend(chunk_entities)
+                    all_relationships.extend(chunk_relationships)
+                    
+                    logger.info(f"✅ Chunk {chunk_idx + 1}: {len(chunk_entities)} entities, {len(chunk_relationships)} relationships")
+                    break
+                else:
+                    logger.info(f"Chunk {chunk_idx + 1} attempt {attempt + 1}/{max_retries}: Validation failed, retrying")
+        
+        # Deduplicate entities across chunks (same entity name = same entity)
+        unique_entities = {}
+        for entity in all_entities:
+            entity_key = entity.get("text", "").lower().strip()
+            if entity_key and entity_key not in unique_entities:
+                unique_entities[entity_key] = entity
+            elif entity_key:
+                # Merge information from duplicate entities
+                existing = unique_entities[entity_key]
+                existing["confidence"] = max(existing.get("confidence", 0), entity.get("confidence", 0))
+                # Combine descriptions
+                existing_desc = existing.get("content", "")
+                new_desc = entity.get("content", "")
+                if new_desc and new_desc not in existing_desc:
+                    existing["content"] = f"{existing_desc}; {new_desc}".strip("; ")
+        
+        # Deduplicate relationships (same startNode+endNode+type = same relationship)
+        unique_relationships = {}
+        for rel in all_relationships:
+            rel_key = f"{rel.get('startNode', '').lower()}||{rel.get('endNode', '').lower()}||{rel.get('type', '').lower()}"
+            if rel_key and rel_key not in unique_relationships:
+                unique_relationships[rel_key] = rel
+            elif rel_key:
+                # Keep highest confidence relationship
+                existing = unique_relationships[rel_key]
+                if rel.get("confidence", 0) > existing.get("confidence", 0):
+                    unique_relationships[rel_key] = rel
+        
+        final_entities = list(unique_entities.values())
+        final_relationships = list(unique_relationships.values())
+        
+        logger.info(f"Final results for {file_name}: {len(final_entities)} unique entities, {len(final_relationships)} unique relationships")
+        
+        return {
+            "entities": final_entities,
+            "relationships": final_relationships,
+            "file": file_name,
+            "structure": {"section": "full_analysis"},
+            "chunks_processed": len(chunks),
+            "total_content_length": len(processed_content)
+        }
+    
+
+    
+    # STEP 9: Single File Processing
+    def process_md_file(self, md_file_path: str) -> Dict[str, Any]:
+        logger.info(f"Processing: {md_file_path}")
+        
+        try:
+            # Read file content
+            with open(md_file_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            file_name = os.path.basename(md_file_path)
+            
+            # Extract entities and relationships using LLM-only approach
+            llm_data = self.extract_entities_with_llm(content, file_name)
+            
+            # Use LLM data - create nodes and relationships from validated data
+            entities_added = 0
+            relationships_added = 0
+            
+            # Check if quota was exceeded during extraction
+            quota_exceeded = llm_data.get("quota_exceeded", False)
+            if quota_exceeded:
+                return {
+                    "file": file_name,
+                    "status": "quota_exceeded",
+                    "entities_extracted": len(llm_data.get("entities", [])),
+                    "unique_entities_added": 0,
+                    "relationships_generated": 0,
+                    "processed_at": datetime.now().isoformat(),
+                    "error": "API quota exceeded during processing"
+                }
+            
+            # Process entities from LLM
+            for entity in llm_data.get("entities", []):
+                entity_text = entity["text"]
+                semantic_key = entity_text.lower().strip()
+                
+                # Add to global registry if new
+                if semantic_key not in self.global_entity_registry:
+                    # Use LLM data directly
+                    entity["id"] = str(uuid.uuid4())
+                    entity["source_file"] = file_name
+                    
+                    self.global_entity_registry[semantic_key] = entity
+                    self.graph_data["nodes"].append(entity)
+                    entities_added += 1
+            
+            # Process relationships from LLM 
+            for rel in llm_data.get("relationships", []):
+                # Apply confidence threshold filtering
+                rel_confidence = rel.get("confidence", 0.0)
+                if rel_confidence < self.min_confidence:
+                    continue  # Skip low-confidence relationships
+                
+                start_text = rel["startNode"].lower().strip()
+                end_text = rel["endNode"].lower().strip()
+                
+                # Only create if both entities exist
+                if start_text in self.global_entity_registry and end_text in self.global_entity_registry:
+                    # Use original relationship type without sanitization
+                    original_type = rel["type"]
+                    
+                    # Create clean relationship with only Neo4j fields
+                    clean_rel = {
+                        "id": str(uuid.uuid4()),
+                        "startNode": self.global_entity_registry[start_text]["id"],
+                        "endNode": self.global_entity_registry[end_text]["id"],
+                        "type": original_type,  # Use original type preserving semantic meaning
+                        "description": rel.get("description", ""),
+                        "evidence": rel.get("evidence", ""),
+                        "confidence": rel_confidence,
+                        "chunk_id": rel.get("chunk_id", 0),
+                        "source_chunk": rel.get("source_chunk", ""),
+                        "source_file": file_name
+                    }
+                    
+                    self.graph_data["relationships"].append(clean_rel)
+                    relationships_added += 1
+            
+            result = {
+                "file": file_name,
+                "status": "success",
+                "entities_extracted": len(llm_data.get("entities", [])),
+                "unique_entities_added": entities_added,
+                "relationships_generated": relationships_added,
+                "processed_at": datetime.now().isoformat()
+            }
+            
+            self.processed_files += 1
+            logger.info(f"✅ Processed {file_name}: {entities_added} new entities, {relationships_added} relationships")
+            return result
+            
+        except Exception as e:
+            logger.error(f"❌ Error processing {md_file_path}: {e}")
+            return {
+                "file": os.path.basename(md_file_path),
+                "status": "error",
+                "error": str(e),
+                "processed_at": datetime.now().isoformat()
+            }
+    
+    # STEP 10: Batch File Processing
+    def process_all_md_files(self, input_dir: str = None, output_path: str = None) -> Dict[str, Any]:
+        if input_dir is None:
+            input_dir = "workspace/processed"
+        if output_path is None:
+            output_path = os.path.join("workspace/graph_data", "graph-data-initial.json")
+        
+        # Clean the graph folder before starting fresh processing
+        graph_dir = os.path.dirname(output_path)
+        self.clean_graph_folder(graph_dir)
+        
+        input_path = Path(input_dir)
+        md_files = list(input_path.glob("**/*.md"))  # Include subdirectories
+        
+        # Ensure output directory exists
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        
+        if not md_files:
+            logger.warning(f"No markdown files found in {input_dir}")
+            return {"status": "no_files", "message": "No markdown files found"}
+        
+        logger.info(f"Found {len(md_files)} markdown files to process")
+        
+        # Reset data structures for a clean batch processing
+        self.graph_data = {"nodes": [], "relationships": []}
+        self.global_entity_registry = {}  # Reset global registry
+        self.processed_files = 0
+        
+        logger.info(f"🚀 Starting document processing with Neo4j format output ({self.llm_provider.upper()})...")
+        
+        # Process files with progress tracking
+        results = []
+        processed_successfully = []
+        failed_files = []
+        quota_exceeded_files = []
+        start_time = time.time()
+        
+        for i, md_file in enumerate(md_files, 1):
+            file_start_time = time.time()
+            logger.info(f"Processing file {i}/{len(md_files)}: {md_file.name}")
+            
+            # Track registry size before processing
+            initial_registry_size = len(self.global_entity_registry)
+            initial_relationship_count = len(self.graph_data["relationships"])
+            
+            # Process the file
+            result = self.process_md_file(str(md_file))
+            results.append(result)
+            
+            # Track file status for detailed logging
+            file_status = result.get("status", "unknown")
+            if file_status == "success":
+                processed_successfully.append(md_file.name)
+            elif file_status == "quota_exceeded":
+                quota_exceeded_files.append(md_file.name)
+                logger.warning(f"🚫 QUOTA EXCEEDED - Stopping batch processing at file {i}/{len(md_files)}")
+                break  # Stop processing when quota exceeded
+            else:
+                failed_files.append((md_file.name, result.get("error", "Unknown error")))
+            
+            # Calculate processing metrics
+            file_time = time.time() - file_start_time
+            new_entities = len(self.global_entity_registry) - initial_registry_size
+            new_relationships = len(self.graph_data["relationships"]) - initial_relationship_count
+            
+            # Show detailed progress information
+            logger.info(f" File processed in {file_time:.2f}s: {new_entities} new entities, {new_relationships} relationships")
+            
+            # Show batch progress at regular intervals
+            if i % 5 == 0 or i == len(md_files):
+                successful_so_far = sum(1 for r in results if r.get("status") == "success")
+                elapsed = time.time() - start_time
+                avg_time = elapsed / i
+                remaining = avg_time * (len(md_files) - i)
+                
+                logger.info(f"Progress: {i}/{len(md_files)} files ({successful_so_far} successful)")
+                logger.info(f" Current stats: {len(self.global_entity_registry)} unique entities, {len(self.graph_data['relationships'])} relationships")
+                logger.info(f"Time elapsed: {elapsed:.1f}s (avg {avg_time:.1f}s per file, ~{remaining:.1f}s remaining)")
+        
+        # Generate comprehensive summary with detailed tracking
+        elapsed = time.time() - start_time
+        successful = len(processed_successfully)
+        quota_exceeded = len(quota_exceeded_files)
+        failed = len(failed_files)
+        unique_entities = len(self.global_entity_registry)
+        
+        # Save detailed processing lists
+        self._save_processing_logs(processed_successfully, quota_exceeded_files, failed_files, output_path)
+        
+        # Count entity types
+        entity_types = {}
+        for entity_info in self.global_entity_registry.values():
+            entity_type = entity_info["type"]
+            entity_types[entity_type] = entity_types.get(entity_type, 0) + 1
+        
+        # Count relationship types
+        relationship_types = {}
+        for rel in self.graph_data["relationships"]:
+            rel_type = rel["type"]
+            relationship_types[rel_type] = relationship_types.get(rel_type, 0) + 1
+        
+        summary = {
+            "status": "completed",
+            "total_files": len(md_files),
+            "successful": successful,
+            "quota_exceeded": quota_exceeded,
+            "failed": failed,
+            "unique_entities": unique_entities,
+            "total_relationships": len(self.graph_data["relationships"]),
+            "entity_types": entity_types,
+            "relationship_types": relationship_types,
+            "processing_time_seconds": elapsed,
+            "average_time_per_file": elapsed / len(md_files) if md_files else 0,
+            "model": self.model_name,
+            "llm_provider": self.llm_provider,
+            "processed_at": datetime.now().isoformat()
+        }
+        
+        logger.info(f"✅ Processing complete in {elapsed:.1f}s: {successful}/{len(md_files)} files successful")
+        if quota_exceeded > 0:
+            logger.warning(f"🚫 {quota_exceeded} files hit quota limit")
+        if failed > 0:
+            logger.error(f"❌ {failed} files failed with errors")
+        logger.info(f"Final stats: {unique_entities} unique entities, {len(self.graph_data['relationships'])} relationships")
+        
+        # Log entity and relationship type breakdown
+        logger.info("Entity types:")
+        for entity_type, count in sorted(entity_types.items(), key=lambda x: x[1], reverse=True)[:10]:
+            logger.info(f"  - {entity_type}: {count}")
+        
+        logger.info("Relationship types:")
+        for rel_type, count in sorted(relationship_types.items(), key=lambda x: x[1], reverse=True)[:10]:
+            logger.info(f"  - {rel_type}: {count}")
+            
+            return summary
+    
+    # STEP 10.5: Processing Logs Tracking
+    def _save_processing_logs(self, successful_files: List[str], quota_exceeded_files: List[str], failed_files: List[tuple], output_path: str):
+        try:
+            output_dir = Path(output_path).parent
+            
+            # Save successfully processed files
+            with open(output_dir / "processed_successfully.txt", 'w', encoding='utf-8') as f:
+                f.write(f"# Successfully Processed Files ({len(successful_files)} total)\n")
+                f.write(f"# Generated: {datetime.now().isoformat()}\n\n")
+                for file_name in successful_files:
+                    f.write(f"{file_name}\n")
+            
+            # Save quota exceeded files
+            if quota_exceeded_files:
+                with open(output_dir / "quota_exceeded_files.txt", 'w', encoding='utf-8') as f:
+                    f.write(f"# Files That Hit Quota Limit ({len(quota_exceeded_files)} total)\n")
+                    f.write(f"# Generated: {datetime.now().isoformat()}\n\n")
+                    for file_name in quota_exceeded_files:
+                        f.write(f"{file_name}\n")
+            
+            # Save failed files with errors
+            if failed_files:
+                with open(output_dir / "failed_files.txt", 'w', encoding='utf-8') as f:
+                    f.write(f"# Files That Failed Processing ({len(failed_files)} total)\n")
+                    f.write(f"# Generated: {datetime.now().isoformat()}\n\n")
+                    for file_name, error in failed_files:
+                        f.write(f"{file_name}: {error}\n")
+                        
+            logger.info(f"📋 Processing logs saved to {output_dir}")
+            
+        except Exception as e:
+            logger.error(f"❌ Failed to save processing logs: {e}")
+    
+    # STEP 11: Graph Data Output
+    def save_graph_data(self, output_path: str = None) -> bool:
+        if output_path is None:
+            output_path = os.path.join("workspace/graph_data", "graph-data-initial.json")
+        try:
+            # Ensure output directory exists
+            output_dir = Path(output_path).parent
+            output_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Compile final data from global entity registry
+            final_nodes = []
+            
+            for semantic_key, entity_info in self.global_entity_registry.items():
+                entity_id = entity_info["id"]
+                
+                # Create Neo4j node
+                node = {
+                    "id": entity_id,
+                    "elementId": entity_id,
+                    "labels": [entity_info["type"]], 
+                    "properties": {
+                        "name": entity_info["text"],
+                        "content": entity_info.get("content", ""),
+                        "source": entity_info.get("source_file", ""),
+                        "confidence": entity_info["confidence"],
+                        "created_date": datetime.now().strftime("%Y-%m-%d"),
+                        "extraction_method": self.llm_provider
+                    }
+                }
+                final_nodes.append(node)
+            
+            # Use relationships
+            final_relationships = self.graph_data["relationships"]
+            
+            # Prepare final graph data
+            final_graph = {
+                "nodes": final_nodes,
+                "relationships": final_relationships,
+                "metadata": {
+                    "node_count": len(final_nodes),
+                    "relationship_count": len(final_relationships),
+                    "generated_at": datetime.now().isoformat(),
+                    "generator": "Allycat GraphBuilder",
+                    "llm_provider": self.llm_provider,
+                    "model": self.model_name,
+                    "format_version": "neo4j-2025"
+                }
+            }
+            
+            # Save final graph data
+            with open(output_path, 'w', encoding='utf-8') as f:
+                json.dump(final_graph, f, indent=2, ensure_ascii=False)
+            
+            # Calculate final output size
+            output_size = os.path.getsize(output_path)
+            output_size_mb = output_size / (1024 * 1024)
+            
+            logger.info(f"✅ Neo4j graph data saved to {output_path} ({output_size_mb:.2f} MB)")
+            logger.info(f"Final stats: {len(final_nodes)} nodes, {len(final_relationships)} relationships")
+            return True
+            
+        except Exception as e:
+            logger.error(f"❌ Error saving graph data: {e}")
+            return False
+    
+# STEP 12: Main Entry Point
+def main():
+    """Main function to run the content analysis pipeline."""
+    logger.info(" Starting Content Analysis Pipeline (Cloud-based APIs)")
+    
+    # Choose LLM provider from environment or default to cerebras
+    llm_provider = os.getenv("GRAPH_LLM_PROVIDER", "cerebras").lower()
+    logger.info(f" Using LLM provider: {llm_provider.upper()}")
+    
+    # Validate provider choice
+    valid_providers = ["cerebras", "gemini"]
+    if llm_provider not in valid_providers:
+        logger.warning(f"⚠️ Invalid provider '{llm_provider}'. Using 'cerebras' (default)")
+        llm_provider = "cerebras"
+    
+    try:
+        analyzer = GraphBuilder(llm_provider=llm_provider)
+        
+        # Normal processing
+        summary = analyzer.process_all_md_files()
+        
+        if summary["status"] == "no_files":
+            logger.warning("⚠️ No files to process")
+            return 1
+        
+        if analyzer.save_graph_data():
+            logger.info("✅ Content Analysis completed successfully!")
+            logger.info(f" Results: {summary['successful']}/{summary['total_files']} files processed")
+            logger.info(f"Graph: {summary['unique_entities']} nodes, {summary['total_relationships']} relationships")
+            logger.info(f"Model used: {analyzer.model_name} via {llm_provider.upper()}")
+            return 0
+        else:
+            logger.error("❌ Failed to save graph data")
+            return 1
+            
+    except Exception as e:
+        logger.error(f"❌ Pipeline failed: {e}")
+        return 1
+
+if __name__ == "__main__":
+    exit(main())

2b_process_graph_phase2.py

@@ -0,0 +1,427 @@
+"""
+Phase 2: Community Detection using Leiden Algorithm
+Loads graph-data-initial.json, runs community detection, saves graph-data-phase-2.json
+"""
+
+import json
+import logging
+import os
+import time
+from pathlib import Path
+from typing import Dict, Any
+from collections import defaultdict
+
+import networkx as nx
+import igraph as ig
+import leidenalg
+import traceback
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+class GraphBuilderPhase2:
+    """Phase 2: Detect communities using graph algorithms (NetworkX + Leiden)"""
+    
+    def __init__(self):
+        """Initialize Phase 2 processor"""
+        self.graph_data = None
+        self.nx_graph = None
+        self.community_result = None
+        self.community_stats = None
+        self.centrality_metrics = None
+        
+        # Configuration from environment or defaults
+        self.min_community_size = int(os.getenv("GRAPH_MIN_COMMUNITY_SIZE", "5"))
+        self.leiden_resolution = float(os.getenv("GRAPH_LEIDEN_RESOLUTION", "1.0"))
+        self.leiden_iterations = int(os.getenv("GRAPH_LEIDEN_ITERATIONS", "-1"))  # -1 = until convergence
+        self.leiden_seed = int(os.getenv("GRAPH_LEIDEN_SEED", "42"))
+        
+        logger.info("✅ Phase 2 Initialized: Community Detection")
+        logger.info(f"   - Min Community Size: {self.min_community_size}")
+        logger.info(f"   - Leiden Resolution: {self.leiden_resolution}")
+    
+    # STEP 1: Load Graph Data from Phase 1
+    def load_graph_data(self, input_path: str = None) -> bool:
+        """Load graph data from the specified JSON file."""
+        if input_path is None:
+            input_path = "workspace/graph_data/graph-data-initial.json"
+        
+        logger.info(f"Loading graph data from {input_path}...")
+        
+        try:
+            input_file = Path(input_path)
+            if not input_file.exists():
+                logger.error(f"❌ Input file not found: {input_path}")
+                logger.warning("   Please run Phase 1 (2b_process_graph_phase1.py) to generate the graph data.")
+                return False
+            
+            with open(input_file, 'r', encoding='utf-8') as f:
+                self.graph_data = json.load(f)
+            
+            node_count = len(self.graph_data.get("nodes", []))
+            rel_count = len(self.graph_data.get("relationships", []))
+            
+            logger.info(f"   - Found {node_count} nodes and {rel_count} relationships")
+            
+            if node_count == 0:
+                logger.error("❌ Graph data is empty. Cannot proceed.")
+                return False
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"❌ Error loading graph data: {e}")
+            return False
+    
+    # STEP 2: Build NetworkX Graph
+    def _build_networkx_graph(self) -> nx.Graph:
+        """Convert graph_data JSON to NetworkX graph for analysis"""
+        logger.info("Building NetworkX graph from JSON data...")
+        
+        G = nx.Graph()
+        
+        # Add nodes with attributes
+        for node in self.graph_data["nodes"]:
+            node_id = node["id"]
+            properties = node.get("properties", {})
+            
+            G.add_node(
+                node_id,
+                name=properties.get("name", ""),
+                type=node.get("labels", ["Unknown"])[0],
+                description=properties.get("content", ""),
+                source=properties.get("source", ""),
+                confidence=properties.get("confidence", 0.0)
+            )
+        
+        # Add edges with attributes
+        for rel in self.graph_data["relationships"]:
+            start_node = rel.get("startNode")
+            end_node = rel.get("endNode")
+            
+            # Only add edge if both nodes exist
+            if start_node in G.nodes() and end_node in G.nodes():
+                G.add_edge(
+                    start_node,
+                    end_node,
+                    type=rel.get("type", "RELATED_TO"),
+                    evidence=rel.get("evidence", ""),
+                    confidence=rel.get("confidence", 0.0)
+                )
+        
+        logger.info(f"✅ Built NetworkX graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges")
+        
+        # Log basic graph statistics
+        if G.number_of_nodes() > 0:
+            density = nx.density(G)
+            logger.info(f"📊 Graph density: {density:.4f}")
+            
+            if G.number_of_edges() > 0:
+                avg_degree = sum(dict(G.degree()).values()) / G.number_of_nodes()
+                logger.info(f"📊 Average degree: {avg_degree:.2f}")
+        
+        return G
+    
+    # STEP 3: Convert to igraph for Leiden
+    def _convert_to_igraph(self, G: nx.Graph) -> ig.Graph:
+        """Convert NetworkX graph to igraph for Leiden algorithm"""
+        logger.info("🔄 Converting to igraph format for Leiden algorithm...")
+        
+        # Create mapping from node IDs to indices
+        node_list = list(G.nodes())
+        node_to_idx = {node: idx for idx, node in enumerate(node_list)}
+        
+        # Create edge list with indices
+        edges = [(node_to_idx[u], node_to_idx[v]) for u, v in G.edges()]
+        
+        # Create igraph
+        ig_graph = ig.Graph(n=len(node_list), edges=edges, directed=False)
+        
+        # Add node attributes
+        ig_graph.vs["name"] = [G.nodes[node].get("name", "") for node in node_list]
+        ig_graph.vs["node_id"] = node_list
+        
+        logger.info(f"✅ Converted to igraph: {ig_graph.vcount()} vertices, {ig_graph.ecount()} edges")
+        
+        return ig_graph
+    
+    # STEP 4: Run Leiden Algorithm
+    def _run_leiden_algorithm(self, ig_graph: ig.Graph) -> Dict[str, Any]:
+        """Run Leiden algorithm for community detection"""
+        logger.info("🔍 Running Leiden community detection algorithm...")
+        logger.info(f"Parameters: resolution={self.leiden_resolution}, iterations={self.leiden_iterations}, seed={self.leiden_seed}")
+        
+        start_time = time.time()
+        
+        try:
+            # Run Leiden algorithm
+            partition = leidenalg.find_partition(
+                ig_graph,
+                leidenalg.ModularityVertexPartition,
+                n_iterations=self.leiden_iterations,
+                seed=self.leiden_seed
+            )
+            
+            # Extract community assignments
+            community_assignments = {}
+            for idx, community_id in enumerate(partition.membership):
+                node_id = ig_graph.vs[idx]["node_id"]
+                community_assignments[node_id] = community_id
+            
+            # Calculate statistics
+            num_communities = len(set(partition.membership))
+            modularity = partition.modularity
+            
+            elapsed = time.time() - start_time
+            
+            logger.info(f"✅ Leiden algorithm completed in {elapsed:.2f}s")
+            logger.info(f"Detected {num_communities} communities")
+            logger.info(f"Modularity score: {modularity:.4f}")
+            
+            return {
+                "assignments": community_assignments,
+                "num_communities": num_communities,
+                "modularity": modularity,
+                "algorithm": "Leiden",
+                "execution_time": elapsed
+            }
+            
+        except Exception as e:
+            logger.error(f"❌ Leiden algorithm failed: {e}")
+            raise e
+    
+    # STEP 5: Calculate Community Statistics
+    def _calculate_community_stats(self, G: nx.Graph, community_assignments: Dict[str, int]) -> Dict[int, Dict]:
+        """Calculate statistics for each community"""
+        logger.info("Calculating community statistics...")
+        
+        # Group nodes by community
+        communities = defaultdict(list)
+        for node_id, comm_id in community_assignments.items():
+            communities[comm_id].append(node_id)
+        
+        # Calculate stats for each community
+        stats = {}
+        for comm_id, node_ids in communities.items():
+            # Skip very small communities if configured
+            if len(node_ids) < self.min_community_size:
+                logger.debug(f"Skipping small community {comm_id} with {len(node_ids)} members")
+                continue
+            
+            subgraph = G.subgraph(node_ids)
+            
+            stats[comm_id] = {
+                "member_count": len(node_ids),
+                "internal_edges": subgraph.number_of_edges(),
+                "density": nx.density(subgraph) if len(node_ids) > 1 else 0.0,
+                "avg_degree": sum(dict(subgraph.degree()).values()) / len(node_ids) if len(node_ids) > 0 else 0.0,
+                "member_ids": node_ids[:20]  # Store top 20 for summary generation
+            }
+        
+        logger.info(f"Calculated statistics for {len(stats)} communities (filtered by min_size={self.min_community_size})")
+        
+        # Log top 5 largest communities
+        sorted_communities = sorted(stats.items(), key=lambda x: x[1]["member_count"], reverse=True)
+        logger.info("Top 5 largest communities:")
+        for comm_id, stat in sorted_communities[:5]:
+            logger.info(f"   Community {comm_id}: {stat['member_count']} members, {stat['internal_edges']} edges, density={stat['density']:.3f}")
+        
+        return stats
+    
+    # STEP 6: Calculate Centrality Metrics
+    def _calculate_centrality_metrics(self, G: nx.Graph) -> Dict[str, Dict]:
+        """Calculate centrality metrics for all nodes"""
+        logger.info("Calculating node centrality metrics...")
+        
+        start_time = time.time()
+        
+        # Degree centrality (fast, always calculate)
+        degree_centrality = nx.degree_centrality(G)
+        
+        # Betweenness centrality (expensive, only for smaller graphs)
+        if G.number_of_nodes() < 5000:
+            logger.info("   Calculating betweenness centrality...")
+            betweenness_centrality = nx.betweenness_centrality(G, k=min(100, G.number_of_nodes()))
+        else:
+            logger.info("   Skipping betweenness centrality (graph too large)")
+            betweenness_centrality = {node: 0.0 for node in G.nodes()}
+        
+        # Closeness centrality (expensive, only for smaller graphs)
+        if G.number_of_nodes() < 5000:
+            logger.info("Calculating closeness centrality...")
+            closeness_centrality = nx.closeness_centrality(G)
+        else:
+            logger.info("   Skipping closeness centrality (graph too large)")
+            closeness_centrality = {node: 0.0 for node in G.nodes()}
+        
+        # Combine metrics
+        centrality_metrics = {}
+        for node in G.nodes():
+            centrality_metrics[node] = {
+                "degree": G.degree(node),
+                "degree_centrality": degree_centrality.get(node, 0.0),
+                "betweenness_centrality": betweenness_centrality.get(node, 0.0),
+                "closeness_centrality": closeness_centrality.get(node, 0.0)
+            }
+        
+        elapsed = time.time() - start_time
+        logger.info(f"✅ Calculated centrality for {len(centrality_metrics)} nodes in {elapsed:.2f}s")
+        
+        return centrality_metrics
+    
+    # STEP 7: Add Community Data to Nodes
+    def _add_community_data_to_nodes(self, community_assignments: Dict[str, int], centrality_metrics: Dict[str, Dict]) -> None:
+        """Add community_id and centrality metrics to node properties"""
+        logger.info("Adding community assignments and centrality to nodes...")
+        
+        nodes_updated = 0
+        
+        for node in self.graph_data["nodes"]:
+            node_id = node["id"]
+            
+            # Add community_id
+            if node_id in community_assignments:
+                node["properties"]["community_id"] = f"comm-{community_assignments[node_id]}"
+                nodes_updated += 1
+            
+            # Add centrality metrics
+            if node_id in centrality_metrics:
+                metrics = centrality_metrics[node_id]
+                node["properties"]["degree"] = metrics["degree"]
+                node["properties"]["degree_centrality"] = round(metrics["degree_centrality"], 4)
+                node["properties"]["betweenness_centrality"] = round(metrics["betweenness_centrality"], 4)
+                node["properties"]["closeness_centrality"] = round(metrics["closeness_centrality"], 4)
+        
+        logger.info(f"✅ Updated {nodes_updated} nodes with community and centrality data")
+    
+    # STEP 8: Main Processing Entry Point
+    def run_community_detection(self, input_path: str = None, output_path: str = None) -> bool:
+        """Main entry point for Phase 2"""
+        if output_path is None:
+            output_path = "workspace/graph_data/graph-data-phase-2.json"
+        
+        logger.info("🚀 Starting Phase 2: Community Detection")
+        logger.info("=" * 60)
+        
+        start_time = time.time()
+        
+        # Step 1: Load Phase 1 output
+        if not self.load_graph_data(input_path):
+            return False
+        
+        # Step 2: Build NetworkX graph
+        self.nx_graph = self._build_networkx_graph()
+        
+        if self.nx_graph.number_of_nodes() == 0:
+            logger.error("❌ Cannot run community detection on empty graph")
+            return False
+        
+        # Step 3: Convert to igraph
+        ig_graph = self._convert_to_igraph(self.nx_graph)
+        
+        # Step 4: Run Leiden algorithm
+        self.community_result = self._run_leiden_algorithm(ig_graph)
+        
+        # Step 5: Calculate community statistics
+        self.community_stats = self._calculate_community_stats(
+            self.nx_graph, 
+            self.community_result["assignments"]
+        )
+        
+        # Step 6: Calculate centrality metrics
+        self.centrality_metrics = self._calculate_centrality_metrics(self.nx_graph)
+        
+        # Step 7: Add community data to nodes
+        self._add_community_data_to_nodes(
+            self.community_result["assignments"],
+            self.centrality_metrics
+        )
+        
+        # Step 8: Update metadata
+        self.graph_data["metadata"]["phase"] = "community_detection"
+        self.graph_data["metadata"]["community_detection"] = {
+            "algorithm": "Leiden",
+            "num_communities": self.community_result["num_communities"],
+            "modularity_score": round(self.community_result["modularity"], 4),
+            "execution_time_seconds": round(self.community_result["execution_time"], 2),
+            "min_community_size": self.min_community_size,
+            "resolution": self.leiden_resolution
+        }
+        
+        # Step 9: Add community statistics to output
+        self.graph_data["community_stats"] = self.community_stats
+        
+        # Step 10: Save Phase 2 output
+        if self._save_phase2_output(output_path):
+            elapsed = time.time() - start_time
+            logger.info("=" * 60)
+            logger.info(f"✅ Phase 2 completed successfully in {elapsed:.2f}s")
+            logger.info(f"Final stats:")
+            logger.info(f"   - Communities detected: {self.community_result['num_communities']}")
+            logger.info(f"   - Modularity score: {self.community_result['modularity']:.4f}")
+            logger.info(f"   - Nodes with community assignments: {len(self.community_result['assignments'])}")
+            logger.info(f"   - Output saved to: {output_path}")
+            return True
+        else:
+            return False
+    
+    # STEP 9: Save Phase 2 Output
+    def _save_phase2_output(self, output_path: str) -> bool:
+        """Save graph-data-phase-2.json"""
+        try:
+            # Ensure output directory exists
+            output_dir = Path(output_path).parent
+            output_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Save Phase 2 output
+            with open(output_path, 'w', encoding='utf-8') as f:
+                json.dump(self.graph_data, f, indent=2, ensure_ascii=False)
+            
+            # Calculate file size
+            output_size = os.path.getsize(output_path)
+            output_size_mb = output_size / (1024 * 1024)
+            
+            logger.info(f"Saved Phase 2 output: {output_path} ({output_size_mb:.2f} MB)")
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"❌ Error saving Phase 2 output: {e}")
+            return False
+
+
+# STEP 10: Main Entry Point
+def main():
+    """Main function to run Phase 2: Community Detection"""
+    logger.info("🚀 GraphRAG Phase 2: Community Detection")
+    logger.info("   Input: graph-data-initial.json (from Phase 1)")
+    logger.info("   Output: graph-data-phase-2.json")
+    logger.info("")
+    
+    try:
+        # Initialize Phase 2 processor
+        processor = GraphBuilderPhase2()
+        
+        # Run community detection
+        success = processor.run_community_detection()
+        
+        if success:
+            logger.info("")
+            logger.info("✅ Phase 2 completed successfully!")
+            logger.info("Next step: Run Phase 3 (2b_process_graph_phase3.py) for community summarization")
+            return 0
+        else:
+            logger.error("")
+            logger.error("❌ Phase 2 failed")
+            logger.error("   Please check the logs above for details")
+            return 1
+            
+    except Exception as e:
+        logger.error(f"❌ Phase 2 pipeline failed: {e}")
+        logger.error(traceback.format_exc())
+        return 1
+
+
+if __name__ == "__main__":
+    exit(main())

2b_process_graph_phase3.py

@@ -0,0 +1,1096 @@
+"""
+Phase 3: Community Summarization using LLM
+Loads graph-data-phase-2.json, generates summaries, saves graph-data-final.json
+"""
+
+import json
+import logging
+import os
+import time
+from pathlib import Path
+from typing import Dict, Any, List
+from datetime import datetime
+from collections import defaultdict
+
+import networkx as nx
+import openai
+import google.generativeai as genai
+
+# JSON parsing libraries (same as Phase 1)
+import orjson
+from json_repair import repair_json
+
+from my_config import MY_CONFIG
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+class GraphBuilderPhase3:
+    """Phase 3: Generate community summaries using LLM"""
+    
+    def __init__(self, llm_provider: str = "cerebras"):
+        """Initialize Phase 3 processor"""
+        self.llm_provider = llm_provider.lower()
+        self.graph_data = None
+        self.nx_graph = None
+        self.community_assignments = {}
+        self.community_stats = {}
+        
+        # Initialize LLM API based on provider
+        if self.llm_provider == "cerebras":
+            if not MY_CONFIG.CEREBRAS_API_KEY:
+                raise ValueError("CEREBRAS_API_KEY not set")
+            
+            self.cerebras_client = openai.OpenAI(
+                api_key=MY_CONFIG.CEREBRAS_API_KEY,
+                base_url="https://api.cerebras.ai/v1"
+            )
+            self.model_name = "llama-4-scout-17b-16e-instruct"
+            logger.info("🚀 Using Cerebras API")
+            
+        elif self.llm_provider == "gemini":
+            if not MY_CONFIG.GEMINI_API_KEY:
+                raise ValueError("GEMINI_API_KEY not set")
+            
+            genai.configure(api_key=MY_CONFIG.GEMINI_API_KEY)
+            self.model_name = "gemini-1.5-flash"
+            self.gemini_model = genai.GenerativeModel(self.model_name)
+            logger.info("🆓 Using Google Gemini API")
+            
+        else:
+            raise ValueError(f"Invalid provider '{llm_provider}'. Choose: cerebras, gemini")
+        
+        # Initialize embedding model for DRIFT search metadata
+        try:
+            from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+            self.embedding_model = HuggingFaceEmbedding(
+                model_name=MY_CONFIG.EMBEDDING_MODEL
+            )
+            logger.info(f"🔍 Initialized embedding model: {MY_CONFIG.EMBEDDING_MODEL}")
+        except Exception as e:
+            logger.warning(f"⚠️ Embedding model initialization failed: {e}")
+            self.embedding_model = None
+        
+        logger.info("✅ Phase 3 initialized: Community Summarization")
+        logger.info(f"📊 LLM Provider: {self.llm_provider.upper()}, Model: {self.model_name}")
+    
+    # STEP 1: Load Phase 2 Output
+    def load_graph_data(self, input_path: str = None) -> bool:
+        """Load graph-data-phase-2.json from Phase 2"""
+        if input_path is None:
+            input_path = "workspace/graph_data/graph-data-phase-2.json"
+        
+        try:
+            input_file = Path(input_path)
+            if not input_file.exists():
+                logger.error(f"❌ Input file not found: {input_path}")
+                logger.error("   Please run Phase 2 (2b_process_graph_phase2.py) first")
+                return False
+            
+            with open(input_file, 'r', encoding='utf-8') as f:
+                self.graph_data = json.load(f)
+            
+            node_count = len(self.graph_data.get("nodes", []))
+            rel_count = len(self.graph_data.get("relationships", []))
+            
+            # Verify Phase 2 was completed
+            if self.graph_data.get("metadata", {}).get("phase") != "community_detection":
+                logger.error("❌ Input file is not from Phase 2 (community_detection)")
+                return False
+            
+            logger.info(f"📂 Loaded graph-data-phase-2.json: {node_count} nodes, {rel_count} relationships")
+            
+            # Load community stats
+            self.community_stats = self.graph_data.get("community_stats", {})
+            num_communities = len(self.community_stats)
+            logger.info(f"📊 Found {num_communities} communities to summarize")
+            
+            if num_communities == 0:
+                logger.error("❌ No communities found in Phase 2 output")
+                return False
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"❌ Error loading graph data: {e}")
+            return False
+    
+    # STEP 2: Build NetworkX Graph
+    def _build_networkx_graph(self) -> nx.Graph:
+        """Rebuild NetworkX graph from JSON data"""
+        logger.info("🔨 Building NetworkX graph from JSON data...")
+        
+        G = nx.Graph()
+        
+        # Add nodes with attributes
+        for node in self.graph_data["nodes"]:
+            node_id = node["id"]
+            properties = node.get("properties", {})
+            
+            G.add_node(
+                node_id,
+                name=properties.get("name", ""),
+                type=node.get("labels", ["Unknown"])[0],
+                description=properties.get("content", ""),
+                community_id=properties.get("community_id", ""),
+                degree_centrality=properties.get("degree_centrality", 0.0)
+            )
+        
+        # Add edges
+        for rel in self.graph_data["relationships"]:
+            start_node = rel.get("startNode")
+            end_node = rel.get("endNode")
+            
+            if start_node in G.nodes() and end_node in G.nodes():
+                G.add_edge(start_node, end_node)
+        
+        logger.info(f"✅ Built NetworkX graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges")
+        
+        return G
+    
+    # STEP 3: Extract Community Assignments
+    def _extract_community_assignments(self) -> Dict[str, int]:
+        """Extract community assignments from node properties"""
+        logger.info("📋 Extracting community assignments from nodes...")
+        
+        assignments = {}
+        
+        for node in self.graph_data["nodes"]:
+            node_id = node["id"]
+            comm_id_str = node.get("properties", {}).get("community_id", "")
+            
+            if comm_id_str and comm_id_str.startswith("comm-"):
+                try:
+                    comm_id = int(comm_id_str.replace("comm-", ""))
+                    assignments[node_id] = comm_id
+                except ValueError:
+                    logger.warning(f"Invalid community_id format: {comm_id_str}")
+        
+        logger.info(f"✅ Extracted {len(assignments)} community assignments")
+        
+        return assignments
+    
+    # STEP 4: LLM Inference Methods
+    def _cerebras_inference(self, system_prompt: str, user_prompt: str) -> str:
+        """Call Cerebras API for inference"""
+        try:
+            # Calculate dynamic parameters based on community size and complexity
+            total_nodes = self.nx_graph.number_of_nodes() if hasattr(self, 'nx_graph') else 100
+            complexity_factor = min(1.0, total_nodes / 1000)
+            
+            # Adaptive temperature: higher for complex graphs to encourage creativity
+            dynamic_temperature = round(0.1 + (complexity_factor * 0.4), 2)  # Range: 0.1-0.5
+            
+            # Adaptive tokens: more for larger/complex summaries
+            dynamic_tokens = int(300 + (complexity_factor * 400))  # Range: 300-700
+            
+            response = self.cerebras_client.chat.completions.create(
+                model=self.model_name,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt}
+                ],
+                temperature=dynamic_temperature,
+                max_tokens=dynamic_tokens
+            )
+            
+            if not response or not response.choices or not response.choices[0].message.content:
+                raise ValueError("Empty response from Cerebras")
+            
+            return response.choices[0].message.content.strip()
+            
+        except Exception as e:
+            logger.error(f"Cerebras inference error: {e}")
+            raise e
+    
+    def _gemini_inference(self, system_prompt: str, user_prompt: str) -> str:
+        """Call Gemini API for inference"""
+        try:
+            # Calculate dynamic generation config based on graph complexity
+            total_nodes = self.nx_graph.number_of_nodes() if hasattr(self, 'nx_graph') else 100
+            complexity_factor = min(1.0, total_nodes / 1000)
+            
+            # Adaptive temperature and tokens for Gemini
+            dynamic_temperature = round(0.1 + (complexity_factor * 0.4), 2)
+            dynamic_tokens = int(300 + (complexity_factor * 400))
+            
+            generation_config = {
+                "temperature": dynamic_temperature,
+                "max_output_tokens": dynamic_tokens,
+                "candidate_count": 1
+            }
+            
+            combined_prompt = f"{system_prompt}\n\n{user_prompt}"
+            response = self.gemini_model.generate_content(
+                combined_prompt,
+                generation_config=generation_config
+            )
+            
+            if not response or not response.text:
+                raise ValueError("Empty response from Gemini")
+            
+            return response.text.strip()
+            
+        except Exception as e:
+            logger.error(f"Gemini inference error: {e}")
+            raise e
+    
+    # STEP 5: Generate Community Summaries
+    def _generate_community_summaries(self) -> Dict[int, str]:
+        """Generate LLM summaries for each community"""
+        logger.info("📝 Generating community summaries with LLM...")
+        logger.info(f"   Total communities to summarize: {len(self.community_stats)}")
+        
+        summaries = {}
+        
+        # Group nodes by community
+        communities = defaultdict(list)
+        for node_id, comm_id in self.community_assignments.items():
+            communities[comm_id].append(node_id)
+        
+        start_time = time.time()
+        
+        for idx, (comm_id_str, stats) in enumerate(self.community_stats.items(), 1):
+            comm_id = int(comm_id_str)
+            
+            logger.info(f"   Processing community {idx}/{len(self.community_stats)}: comm-{comm_id} ({stats['member_count']} members)")
+            
+            # Get top entities by centrality
+            node_ids = communities[comm_id]
+            subgraph = self.nx_graph.subgraph(node_ids)
+            
+            # Get nodes sorted by degree centrality
+            centrality = nx.degree_centrality(subgraph)
+            top_nodes = sorted(centrality.items(), key=lambda x: x[1], reverse=True)[:15]
+            
+            # Prepare entity information for LLM
+            entity_info = []
+            for node_id, _ in top_nodes:
+                node_data = self.nx_graph.nodes[node_id]
+                entity_info.append({
+                    "name": node_data.get("name", "Unknown"),
+                    "type": node_data.get("type", "Unknown"),
+                    "description": node_data.get("description", "")[:150]  # Limit length
+                })
+            
+            # Create LLM prompt
+            # Senior-developer style system/user prompts with strict output schema
+            # Calculate dynamic topic count based on community size
+            topic_count = max(2, min(5, stats['member_count'] // 3))  # Scale with community size
+            
+            system_prompt = (
+                "You are a specialized knowledge graph summarization assistant. Your task is to analyze community "
+                "structures and generate comprehensive summaries for graph-based retrieval systems.\n\n"
+                "CONSTITUTIONAL AI PRINCIPLES:\n"
+                "1. Content-Adaptive: Generate summaries based on actual community composition and statistics\n"
+                "2. Context-Aware: Consider entity relationships and community density in summarization\n"
+                "3. Quality-First: Prioritize accuracy and relevance over brevity\n"
+                "4. Structured Output: Ensure consistent JSON format for programmatic consumption\n\n"
+                "SUMMARIZATION GUIDELINES:\n"
+                "- Analyze entity types, relationships, and community structure\n"
+                "- Identify key themes and concepts that define this community\n"
+                "- Generate topics that capture semantic meaning, not just entity names\n"
+                "- Assess confidence based on data completeness and coherence\n"
+                "- Use neutral, factual tone suitable for technical documentation"
+            )
+
+            user_prompt = (
+                f"Analyze the following community data and generate a structured summary.\n\n"
+                f"COMMUNITY STATISTICS:\n"
+                f"- Total Members: {stats['member_count']}\n"
+                f"- Internal Connections: {stats['internal_edges']}\n"
+                f"- Community Density: {stats['density']:.3f}\n"
+                f"- Connectivity Strength: {'High' if stats['density'] > 0.1 else 'Medium' if stats['density'] > 0.05 else 'Low'}\n\n"
+                f"TOP ENTITIES (name, type, description):\n{json.dumps(entity_info, indent=2)}\n\n"
+                f"OUTPUT FORMAT (strict JSON):\n"
+                f"{{\n"
+                f"    \"summary\": \"2-3 sentence comprehensive summary of community purpose and characteristics\",\n"
+                f"    \"primary_topics\": [\"topic_1\", \"topic_2\", \"topic_{topic_count}\"],\n"
+                f"    \"confidence\": 0.85\n"
+                f"}}\n\n"
+                f"VALIDATION REQUIREMENTS:\n"
+                f"- summary: Must be 2-3 complete sentences describing community focus and key characteristics\n"
+                f"- primary_topics: Array of exactly {topic_count} descriptive phrases (not just entity names)\n"
+                f"- confidence: Float between 0.0-1.0 based on data quality and coherence\n\n"
+                f"IMPORTANT: Respond with ONLY the JSON object. No markdown formatting, no explanations, no code blocks."
+            )
+            
+            # Call LLM for summary
+            try:
+                if self.llm_provider == "gemini":
+                    summary_response = self._gemini_inference(system_prompt, user_prompt)
+                else:  # cerebras
+                    summary_response = self._cerebras_inference(system_prompt, user_prompt)
+                
+                # Parse JSON response
+                parsed_summary = self._parse_summary_response(summary_response, comm_id)
+                if parsed_summary:
+                    summaries[comm_id] = parsed_summary
+                else:
+                    # Fallback to raw response if parsing fails
+                    summaries[comm_id] = summary_response.strip()
+                
+                # Log progress every 10 communities
+                if idx % 10 == 0:
+                    elapsed = time.time() - start_time
+                    avg_time = elapsed / idx
+                    remaining = avg_time * (len(self.community_stats) - idx)
+                    logger.info(f"   Progress: {idx}/{len(self.community_stats)} ({elapsed:.1f}s elapsed, ~{remaining:.1f}s remaining)")
+                
+            except Exception as e:
+                logger.error(f"❌ Failed to generate summary for community {comm_id}: {e}")
+                summaries[comm_id] = f"Community with {stats['member_count']} entities focused on {entity_info[0]['type'] if entity_info else 'various'} topics."
+        
+        elapsed = time.time() - start_time
+        logger.info(f"✅ Generated {len(summaries)} community summaries in {elapsed:.1f}s")
+        
+        return summaries
+    
+    def _parse_summary_response(self, response: str, comm_id: int) -> str:
+        """Parse JSON summary response with fallback to text extraction"""
+        try:
+            # Clean response
+            cleaned_response = response.strip()
+            
+            # Remove markdown formatting
+            if "```json" in cleaned_response:
+                parts = cleaned_response.split("```json")
+                if len(parts) > 1:
+                    json_part = parts[1].split("```")[0].strip()
+                    cleaned_response = json_part
+            elif "```" in cleaned_response:
+                parts = cleaned_response.split("```")
+                if len(parts) >= 3:
+                    cleaned_response = parts[1].strip()
+            
+            # Try to parse JSON
+            try:
+                summary_data = self._smart_json_parse_summary(cleaned_response)
+                if summary_data and isinstance(summary_data, dict):
+                    summary_text = summary_data.get('summary', '')
+                    if summary_text and len(summary_text.strip()) > 10:
+                        return summary_text.strip()
+            except ValueError as e:
+                logger.debug(f"Summary JSON parsing failed for comm-{comm_id}: {e}")
+            except Exception as e:
+                logger.debug(f"Summary JSON parsing unexpected error for comm-{comm_id}: {e}")
+                    
+        except Exception as e:
+            logger.debug(f"Summary JSON parsing failed for comm-{comm_id}: {e}")
+        
+        # Fallback: extract first meaningful sentence
+        try:
+            lines = response.split('\n')
+            for line in lines:
+                line = line.strip()
+                if len(line) > 20 and '.' in line and not line.startswith('{'):
+                    return line
+        except Exception:
+            pass
+        
+        return None
+    
+    def _smart_json_parse_summary(self, json_text: str) -> Dict:
+        """
+        Simple 5-step JSON parsing approach (exactly same as Phase 1)
+        """
+        cleaned_text = json_text.strip()
+        
+        # Step 1: orjson
+        try:
+            result = orjson.loads(cleaned_text.encode('utf-8'))
+            logger.debug("✅ Step 1: orjson succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 1: orjson failed - {e}")
+        
+        # Step 2: json-repair
+        try:
+            repaired = repair_json(cleaned_text)
+            result = orjson.loads(repaired.encode('utf-8'))
+            logger.debug("✅ Step 2: json-repair + orjson succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 2: json-repair failed - {e}")
+        
+        # Step 3: standard json
+        try:
+            result = json.loads(cleaned_text)
+            logger.debug("✅ Step 3: standard json succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 3: standard json failed - {e}")
+        
+        # Step 4: json-repair + standard json
+        try:
+            repaired = repair_json(cleaned_text)
+            result = json.loads(repaired)
+            logger.debug("✅ Step 4: json-repair + standard json succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 4: json-repair + standard json failed - {e}")
+        
+        # Step 5: All failed - this will trigger save failed txt files
+        raise ValueError("All 4 JSON parsing steps failed")
+    
+    # STEP 6: Identify Key Entities
+    def _identify_key_entities(self) -> Dict[int, List[str]]:
+        """Identify key entities in each community based on centrality"""
+        logger.info("🔑 Identifying key entities per community...")
+        
+        key_entities = {}
+        
+        # Group nodes by community
+        communities = defaultdict(list)
+        for node_id, comm_id in self.community_assignments.items():
+            communities[comm_id].append(node_id)
+        
+        for comm_id, node_ids in communities.items():
+            subgraph = self.nx_graph.subgraph(node_ids)
+            
+            # Calculate degree centrality
+            centrality = nx.degree_centrality(subgraph)
+            
+            # Get top 5 entities
+            top_nodes = sorted(centrality.items(), key=lambda x: x[1], reverse=True)[:5]
+            
+            key_entities[comm_id] = [
+                self.nx_graph.nodes[node_id].get("name", "Unknown")
+                for node_id, _ in top_nodes
+            ]
+        
+        logger.info(f"✅ Identified key entities for {len(key_entities)} communities")
+        
+        return key_entities
+    
+    # STEP 7: Create Community Nodes
+    def _create_community_nodes(self, community_summaries: Dict[int, str], key_entities: Dict[int, List[str]]) -> List[Dict]:
+        """Create community nodes for the graph"""
+        logger.info("🏗️ Creating community nodes...")
+        
+        import uuid
+        
+        community_nodes = []
+        
+        for comm_id_str, stats in self.community_stats.items():
+            comm_id = int(comm_id_str)
+            
+            node = {
+                "id": f"community-{uuid.uuid4()}",
+                "elementId": f"community-{uuid.uuid4()}",
+                "labels": ["Community"],
+                "properties": {
+                    "community_id": f"comm-{comm_id}",
+                    "level": 1,
+                    "member_count": stats["member_count"],
+                    "internal_edges": stats["internal_edges"],
+                    "density": round(stats["density"], 4),
+                    "avg_degree": round(stats["avg_degree"], 2),
+                    "summary": community_summaries.get(comm_id, ""),
+                    "key_entities": key_entities.get(comm_id, []),
+                    "created_date": datetime.now().isoformat()
+                }
+            }
+            community_nodes.append(node)
+        
+        logger.info(f"✅ Created {len(community_nodes)} community nodes")
+        
+        return community_nodes
+    
+    # STEP 8: Create IN_COMMUNITY Relationships
+    def _create_in_community_relationships(self, community_nodes: List[Dict]) -> List[Dict]:
+        """Create IN_COMMUNITY relationships linking entities to communities"""
+        logger.info("Creating IN_COMMUNITY relationships...")
+        
+        import uuid
+        
+        # Create mapping from community_id to community node id
+        comm_id_to_node_id = {}
+        for node in community_nodes:
+            comm_id = node["properties"]["community_id"]
+            comm_id_to_node_id[comm_id] = node["id"]
+        
+        relationships = []
+        
+        for entity_id, comm_id in self.community_assignments.items():
+            comm_node_id = comm_id_to_node_id.get(f"comm-{comm_id}")
+            
+            if comm_node_id:
+                # Calculate confidence based on community membership strength
+                entity_node = next((n for n in self.graph_data['nodes'] if n['id'] == entity_id), None)
+                if entity_node:
+                    degree_centrality = entity_node.get('properties', {}).get('degree_centrality', 0.5)
+                    # Higher centrality = higher confidence in community assignment
+                    dynamic_confidence = round(0.6 + (degree_centrality * 0.4), 3)  # Range: 0.6-1.0
+                else:
+                    dynamic_confidence = 0.8  # Default for missing nodes
+                
+                rel = {
+                    "id": f"rel-{uuid.uuid4()}",
+                    "startNode": entity_id,
+                    "endNode": comm_node_id,
+                    "type": "IN_COMMUNITY",
+                    "properties": {
+                        "confidence": dynamic_confidence,
+                        "assigned_date": datetime.now().isoformat()
+                    }
+                }
+                relationships.append(rel)
+        
+        logger.info(f"✅ Created {len(relationships)} IN_COMMUNITY relationships")
+        
+        return relationships
+    
+    # STEP 9: DRIFT Search Metadata Generation
+    def _generate_drift_metadata(self, community_summaries: Dict[int, str], key_entities: Dict[int, List[str]]) -> Dict:
+        """Generate DRIFT search metadata using existing embedding infrastructure"""
+        logger.info("🔍 Generating DRIFT search metadata...")
+        
+        if not self.embedding_model:
+            logger.warning("⚠️ Embedding model not available, skipping DRIFT metadata")
+            return {}
+        
+        # Calculate dynamic values from actual graph data
+        total_communities = len(community_summaries)
+        total_nodes = self.nx_graph.number_of_nodes()
+        total_edges = self.nx_graph.number_of_edges()
+        avg_community_size = sum(self.community_stats.get(str(i), {}).get("member_count", 0) 
+                               for i in community_summaries.keys()) / total_communities if total_communities > 0 else 0
+        graph_density = total_edges / (total_nodes * (total_nodes - 1) / 2) if total_nodes > 1 else 0
+        
+        # Calculate dynamic thresholds based on graph complexity
+        complexity_factor = min(1.0, (total_nodes + total_edges) / 10000)  # Scale 0-1 based on graph size
+        base_confidence = 0.6 + (complexity_factor * 0.3)  # Range: 0.6-0.9
+        base_response_time = 1.0 + (complexity_factor * 3.0)  # Range: 1-4 seconds
+        base_memory = int(20 + (avg_community_size * complexity_factor * 5))  # Scale with size
+        
+        # Adaptive configuration based on graph characteristics
+        max_communities_for_primer = min(total_communities, max(2, total_communities // 4))
+        lightweight_communities = max(1, max_communities_for_primer // 2)
+        standard_communities = max(2, int(max_communities_for_primer // 1.5))
+        comprehensive_communities = max_communities_for_primer
+        
+        # Calculate dynamic iteration counts based on community distribution
+        max_iter = max(2, min(5, int(total_communities / 10) + 2))
+        hyde_count = max(2, min(5, int(avg_community_size / 5) + 2))
+        
+        drift_metadata = {
+            "version": "1.0",
+            "generated_timestamp": datetime.now().isoformat(),
+            "configuration": {
+                "max_iterations": max_iter,
+                "confidence_threshold": round(base_confidence + 0.1, 2),
+                "top_k_communities": max_communities_for_primer,
+                "hyde_expansion_count": hyde_count,
+                "termination_criteria": "confidence_or_max_iterations"
+            },
+            "query_routing_config": {
+                "lightweight_drift": {
+                    "triggers": ["single_entity", "simple_fact", "definition_query"],
+                    "config": {
+                        "primer_communities": int(lightweight_communities), 
+                        "follow_up_iterations": max(1, max_iter - 2), 
+                        "confidence_threshold": round(base_confidence, 2)
+                    }
+                },
+                "standard_drift": {
+                    "triggers": ["multi_entity", "relationship_query", "how_does"],
+                    "config": {
+                        "primer_communities": int(standard_communities), 
+                        "follow_up_iterations": max(1, max_iter - 1), 
+                        "confidence_threshold": round(base_confidence + 0.1, 2)
+                    }
+                },
+                "comprehensive_drift": {
+                    "triggers": ["analyze", "compare", "implications", "strategy"],
+                    "config": {
+                        "primer_communities": int(comprehensive_communities), 
+                        "follow_up_iterations": max_iter, 
+                        "confidence_threshold": round(base_confidence + 0.2, 2)
+                    }
+                }
+            },
+            "performance_monitoring": {
+                "response_time_targets": {
+                    "p50": round(base_response_time * 1.0, 1),
+                    "p95": round(base_response_time * 2.5, 1), 
+                    "p99": round(base_response_time * 5.0, 1)
+                },
+                "resource_tracking": {
+                    "memory_per_query": base_memory, 
+                    "cache_hit_rate_target": round(0.5 + (complexity_factor * 0.3), 2)
+                },
+                "bottleneck_identification": ["community_ranking", "follow_up_generation", "embedding_computation"]
+            },
+            "community_search_index": {},
+            "search_optimization": {
+                "total_communities": total_communities,
+                "avg_community_size": round(avg_community_size, 1),
+                "graph_density": round(graph_density, 6),
+                "total_nodes": total_nodes,
+                "total_edges": total_edges,
+                "max_primer_communities": max_communities_for_primer
+            }
+        }
+        
+        # Process each community
+        for comm_id, summary in community_summaries.items():
+            comm_key = f"comm-{comm_id}"
+            
+            try:
+                # Generate embeddings using existing HuggingFace model
+                summary_embedding = self.embedding_model.get_text_embedding(summary)
+                hyde_embeddings = self._generate_hyde_embeddings(summary)
+                follow_up_questions = self._generate_follow_up_questions(summary, comm_id, key_entities.get(comm_id, []))
+                
+                # Add to search index
+                drift_metadata["community_search_index"][comm_key] = {
+                    "summary": summary,
+                    "key_entities": key_entities.get(comm_id, []),
+                    "embeddings": {
+                        "summary_embedding": summary_embedding,
+                        "hyde_embeddings": hyde_embeddings
+                    },
+                    "follow_up_templates": follow_up_questions,
+                    "statistics": self.community_stats.get(str(comm_id), {})
+                }
+                
+            except Exception as e:
+                logger.warning(f"⚠️ Failed to generate metadata for {comm_key}: {e}")
+                continue
+        
+        logger.info(f"✅ Generated DRIFT metadata for {len(drift_metadata['community_search_index'])} communities")
+        return drift_metadata
+    
+    def _generate_hyde_embeddings(self, community_summary: str) -> List[List[float]]:
+        """Generate HyDE embeddings for enhanced recall"""
+        
+        # Create 3 hypothetical document variations
+        hyde_templates = [
+            f"Research analysis and findings: {community_summary}",
+            f"Technical report and documentation: {community_summary}",
+            f"Business implications and strategic analysis: {community_summary}"
+        ]
+        
+        hyde_embeddings = []
+        for template in hyde_templates:
+            try:
+                embedding = self.embedding_model.get_text_embedding(template)
+                hyde_embeddings.append(embedding)
+            except Exception as e:
+                logger.warning(f"⚠️ HyDE embedding generation failed: {e}")
+                continue
+        
+        return hyde_embeddings
+    
+    def _generate_follow_up_questions(self, community_summary: str, comm_id: int, key_entities: List[str]) -> List[Dict]:
+        """Generate follow-up questions using existing LLM infrastructure"""
+        
+        # Professional system prompt matching Phase 1 style
+        system_prompt = (
+            "You are a specialized DRIFT search question generation assistant. Your task is to analyze community "
+            "summaries and generate targeted follow-up questions for iterative knowledge graph exploration.\n\n"
+            "CONSTITUTIONAL AI PRINCIPLES:\n"
+            "1. Context-Adaptive: Generate questions based on actual community content and entities\n"
+            "2. Search-Aware: Choose appropriate search types to guide query routing optimization\n"
+            "3. Relevance-First: Prioritize questions that expand understanding of community themes\n"
+            "4. Structured Output: Ensure consistent JSON format for programmatic consumption\n\n"
+            "QUESTION GENERATION GUIDELINES:\n"
+            "- Analyze community summary and key entities to identify knowledge gaps\n"
+            "- Generate questions that would reveal additional relevant information\n"
+            "- Use local search for entity-specific queries, relationship for connections, global for themes\n"
+            "- Assign relevance scores based on potential value for understanding the community\n"
+            "- Target entities should guide search focus and retrieval optimization"
+        )
+
+        user_prompt = (
+            f"Analyze the following community data and generate targeted follow-up questions.\n\n"
+            f"COMMUNITY SUMMARY:\n{community_summary}\n\n"
+            f"KEY ENTITIES: {', '.join(key_entities[:5]) if key_entities else 'No specific entities identified'}\n\n"
+            f"TASK: Generate exactly 3 strategic follow-up questions for DRIFT search.\n\n"
+            f"OUTPUT FORMAT (strict JSON):\n"
+            f"[\n"
+            f"    {{\n"
+            f"        \"question\": \"Specific, actionable question about the community\",\n"
+            f"        \"relevance_score\": 0.85,\n"
+            f"        \"search_type\": \"local\",\n"
+            f"        \"target_entities\": [\"entity1\", \"entity2\"]\n"
+            f"    }}\n"
+            f"]\n\n"
+            f"VALIDATION REQUIREMENTS:\n"
+            f"- question: Must be a clear, specific question that expands community understanding\n"
+            f"- relevance_score: Float 0.0-1.0 based on potential value for knowledge expansion\n"
+            f"- search_type: Must be one of 'local', 'relationship', or 'global'\n"
+            f"- target_entities: Array of relevant entity names from the key entities list\n\n"
+            f"IMPORTANT: Respond with ONLY the JSON array. No markdown formatting, no explanations, no code blocks."
+        )
+        
+        try:
+            # Use existing LLM infrastructure
+            if self.llm_provider == "cerebras":
+                response = self._cerebras_inference(system_prompt, user_prompt)
+            else:
+                response = self._gemini_inference(system_prompt, user_prompt)
+            
+            # Parse LLM response to structured questions
+            questions = self._parse_questions_response(response, key_entities)
+            return questions
+            
+        except Exception as e:
+            logger.error(f"❌ Question generation failed for comm-{comm_id}: {e}")
+            return []
+    
+    def _parse_questions_response(self, response: str, key_entities: List[str]) -> List[Dict]:
+        """Parse LLM response into structured questions using robust multi-strategy approach"""
+        try:
+            # Calculate dynamic default relevance based on community statistics
+            total_nodes = self.nx_graph.number_of_nodes() if hasattr(self, 'nx_graph') else 100
+            node_density = min(1.0, total_nodes / 500)  # Scale 0-1
+            default_relevance = round(0.5 + (node_density * 0.4), 2)  # Range: 0.5-0.9
+            max_questions = max(2, min(5, len(key_entities) + 1))  # Adaptive question count
+            
+            # Strategy 1: JSON array extraction with regex
+            try:
+                import re
+                match = re.search(r"(\[\s*\{[\s\S]*?\}\s*\])", response)
+                if match:
+                    json_str = match.group(1)
+                    try:
+                        questions = self._smart_json_parse_questions(json_str)
+                        if questions:
+                            return self._validate_and_normalize_questions(questions, key_entities, default_relevance, max_questions)
+                    except ValueError:
+                        pass  # Continue to next strategy if JSON parsing fails
+            except Exception:
+                pass
+            
+            # Strategy 2: Multiple JSON objects extraction
+            try:
+                import re
+                pattern = r'\{[^{}]*"question"[^{}]*\}'
+                matches = re.findall(pattern, response)
+                if matches:
+                    json_array = "[" + ",".join(matches) + "]"
+                    try:
+                        questions = self._smart_json_parse_questions(json_array)
+                        if questions:
+                            return self._validate_and_normalize_questions(questions, key_entities, default_relevance, max_questions)
+                    except ValueError:
+                        pass  # Continue to next strategy if JSON parsing fails
+            except Exception:
+                pass
+            
+            # Strategy 3: Markdown list extraction
+            try:
+                questions = self._parse_markdown_questions(response, key_entities, default_relevance)
+                if questions:
+                    return self._validate_and_normalize_questions(questions, key_entities, default_relevance, max_questions)
+            except Exception:
+                pass
+            
+            # Strategy 4: Generate default questions based on entities
+            return self._generate_default_questions(key_entities, default_relevance, max_questions)
+            
+        except Exception as e:
+            logger.warning(f"⚠️ All question parsing strategies failed: {e}")
+            return self._generate_default_questions(key_entities, 0.7, 3)
+    
+    def _smart_json_parse_questions(self, json_text: str) -> List[Dict]:
+        """
+        Simple 5-step JSON parsing approach (exactly same as Phase 1)
+        """
+        cleaned_text = json_text.strip()
+        
+        # Step 1: orjson
+        try:
+            result = orjson.loads(cleaned_text.encode('utf-8'))
+            logger.debug("✅ Step 1: orjson succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 1: orjson failed - {e}")
+        
+        # Step 2: json-repair
+        try:
+            repaired = repair_json(cleaned_text)
+            result = orjson.loads(repaired.encode('utf-8'))
+            logger.debug("✅ Step 2: json-repair + orjson succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 2: json-repair failed - {e}")
+        
+        # Step 3: standard json
+        try:
+            result = json.loads(cleaned_text)
+            logger.debug("✅ Step 3: standard json succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 3: standard json failed - {e}")
+        
+        # Step 4: json-repair + standard json
+        try:
+            repaired = repair_json(cleaned_text)
+            result = json.loads(repaired)
+            logger.debug("✅ Step 4: json-repair + standard json succeeded")
+            return result
+        except Exception as e:
+            logger.debug(f"❌ Step 4: json-repair + standard json failed - {e}")
+        
+        # Step 5: All failed - this will trigger save failed txt files
+        raise ValueError("All 4 JSON parsing steps failed")
+    
+    def _parse_markdown_questions(self, response: str, key_entities: List[str], default_relevance: float) -> List[Dict]:
+        """Parse questions from markdown or plain text format"""
+        questions = []
+        
+        # Look for numbered lists or bullet points
+        import re
+        patterns = [
+            r'\d+\.\s*(.+?)(?=\n\d+\.|\n-|\n\*|$)',  # Numbered list
+            r'-\s*(.+?)(?=\n-|\n\*|\n\d+\.|$)',      # Dash list
+            r'\*\s*(.+?)(?=\n\*|\n-|\n\d+\.|$)'      # Asterisk list
+        ]
+        
+        for pattern in patterns:
+            matches = re.findall(pattern, response, re.MULTILINE | re.DOTALL)
+            if matches and len(matches) >= 2:
+                for i, match in enumerate(matches[:5]):  # Max 5 questions
+                    question_text = match.strip().replace('\n', ' ')
+                    if len(question_text) > 10:  # Reasonable question length
+                        search_type = 'global' if any(word in question_text.lower() 
+                                                    for word in ['analyze', 'compare', 'overall', 'trends']) else 'local'
+                        questions.append({
+                            'question': question_text,
+                            'relevance_score': max(0.6, default_relevance - (i * 0.1)),
+                            'search_type': search_type,
+                            'target_entities': key_entities[:2] if key_entities else []
+                        })
+                break
+        
+        return questions
+    
+    def _generate_default_questions(self, key_entities: List[str], default_relevance: float, max_questions: int) -> List[Dict]:
+        """Generate default questions when parsing fails"""
+        if not key_entities:
+            return []
+        
+        # Template questions based on entity analysis
+        question_templates = [
+            ("What is {entity} and what role does it play?", "local"),
+            ("How does {entity} relate to other entities in this community?", "relationship"), 
+            ("What are the key characteristics and properties of {entity}?", "local"),
+            ("What trends or patterns involve {entity}?", "global"),
+            ("How might {entity} impact the broader context?", "global")
+        ]
+        
+        questions = []
+        entities_to_use = key_entities[:max_questions]
+        
+        for i, entity in enumerate(entities_to_use):
+            if i < len(question_templates):
+                template, search_type = question_templates[i]
+                question = template.format(entity=entity)
+                questions.append({
+                    'question': question,
+                    'relevance_score': max(0.6, default_relevance - (i * 0.05)),
+                    'search_type': search_type,
+                    'target_entities': [entity]
+                })
+        
+        return questions
+    
+    def _validate_and_normalize_questions(self, questions: List[Dict], key_entities: List[str], 
+                                        default_relevance: float, max_questions: int) -> List[Dict]:
+        """Validate and normalize question format"""
+        normalized = []
+        
+        for q in questions:
+            if not isinstance(q, dict):
+                continue
+                
+            # Extract question text
+            question = q.get('question') or q.get('q') or q.get('text')
+            if not question or len(str(question).strip()) < 5:
+                continue
+            
+            # Extract and validate relevance score
+            relevance = q.get('relevance_score', default_relevance)
+            try:
+                relevance = float(relevance)
+                if relevance <= 0 or relevance > 1:
+                    relevance = default_relevance
+            except (ValueError, TypeError):
+                relevance = default_relevance
+            
+            # Extract and validate search type
+            search_type = q.get('search_type', 'local')
+            if search_type not in ('local', 'relationship', 'global'):
+                search_type = 'local'
+            
+            # Extract target entities
+            target_entities = q.get('target_entities', [])
+            if not isinstance(target_entities, list):
+                target_entities = []
+            
+            # Ensure we have some target entities
+            if not target_entities and key_entities:
+                target_entities = key_entities[:2]
+            
+            normalized.append({
+                'question': str(question).strip(),
+                'relevance_score': round(relevance, 2),
+                'search_type': search_type,
+                'target_entities': target_entities
+            })
+            
+            if len(normalized) >= max_questions:
+                break
+        
+        return normalized
+    
+    # STEP 10: Main Processing Entry Point
+    def generate_summaries(self, input_path: str = None, output_path: str = None) -> bool:
+        """Main entry point for Phase 3"""
+        if output_path is None:
+            output_path = "workspace/graph_data/graph-data-final.json"
+        
+        logger.info("🚀 Starting Phase 3: Community Summarization")
+        logger.info("=" * 60)
+        
+        start_time = time.time()
+        
+        # Step 1: Load Phase 2 output
+        if not self.load_graph_data(input_path):
+            return False
+        
+        # Step 2: Build NetworkX graph
+        self.nx_graph = self._build_networkx_graph()
+        
+        # Step 3: Extract community assignments
+        self.community_assignments = self._extract_community_assignments()
+        
+        # Step 4: Generate LLM summaries
+        community_summaries = self._generate_community_summaries()
+        
+        # Step 5: Identify key entities
+        key_entities = self._identify_key_entities()
+        
+        # Step 6: Create community nodes
+        community_nodes = self._create_community_nodes(community_summaries, key_entities)
+        
+        # Step 7: Create IN_COMMUNITY relationships
+        community_relationships = self._create_in_community_relationships(community_nodes)
+        
+        # Step 8: Merge everything
+        self.graph_data["nodes"].extend(community_nodes)
+        self.graph_data["relationships"].extend(community_relationships)
+        
+        # Step 9: Add communities section
+        self.graph_data["communities"] = {
+            "algorithm": "Leiden",
+            "total_communities": len(community_summaries),
+            "modularity_score": self.graph_data["metadata"]["community_detection"]["modularity_score"],
+            "summaries": {
+                f"comm-{k}": v for k, v in community_summaries.items()
+            }
+        }
+        
+        # Step 10: Generate DRIFT search metadata
+        drift_metadata = self._generate_drift_metadata(community_summaries, key_entities)
+        if drift_metadata:
+            self.graph_data["drift_search_metadata"] = drift_metadata
+            logger.info("✅ Added DRIFT search metadata to graph data")
+        
+        # Step 11: Clean up temporary data
+        if "community_stats" in self.graph_data:
+            del self.graph_data["community_stats"]
+        
+        # Step 12: Update metadata
+        self.graph_data["metadata"]["phase"] = "final"
+        self.graph_data["metadata"]["entity_count"] = len([n for n in self.graph_data["nodes"] if "Community" not in n["labels"]])
+        self.graph_data["metadata"]["community_count"] = len(community_nodes)
+        self.graph_data["metadata"]["total_node_count"] = len(self.graph_data["nodes"])
+        self.graph_data["metadata"]["total_relationship_count"] = len(self.graph_data["relationships"])
+        
+        # Step 13: Save final output
+        if self._save_final_output(output_path):
+            elapsed = time.time() - start_time
+            logger.info("=" * 60)
+            logger.info(f"✅ Phase 3 completed successfully in {elapsed:.1f}s")
+            logger.info("📊 Final stats:")
+            logger.info(f"   - Total nodes: {len(self.graph_data['nodes'])}")
+            logger.info(f"   - Entity nodes: {self.graph_data['metadata']['entity_count']}")
+            logger.info(f"   - Community nodes: {len(community_nodes)}")
+            logger.info(f"   - Total relationships: {len(self.graph_data['relationships'])}")
+            logger.info(f"   - Communities with summaries: {len(community_summaries)}")
+            logger.info(f"   - Output saved to: {output_path}")
+            return True
+        else:
+            return False
+    
+    # STEP 14: Save Final Output
+    def _save_final_output(self, output_path: str) -> bool:
+        """Save graph-data-final.json with DRIFT search metadata"""
+        try:
+            # Ensure output directory exists
+            output_dir = Path(output_path).parent
+            output_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Save final output
+            with open(output_path, 'w', encoding='utf-8') as f:
+                json.dump(self.graph_data, f, indent=2, ensure_ascii=False)
+            
+            # Calculate file size
+            output_size = os.path.getsize(output_path)
+            output_size_mb = output_size / (1024 * 1024)
+            
+            logger.info(f"💾 Saved final output: {output_path} ({output_size_mb:.2f} MB)")
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"❌ Error saving final output: {e}")
+            return False
+
+
+# STEP 15: Main Entry Point
+def main():
+    """Main function to run Phase 3: Community Summarization with DRIFT Search Metadata"""
+    logger.info("🚀 GraphRAG Phase 3: Community Summarization + DRIFT Search Metadata")
+    logger.info("   Input: graph-data-phase-2.json (from Phase 2)")
+    logger.info("   Output: graph-data-final.json (with DRIFT search metadata)")
+    logger.info("")
+    
+    # Choose LLM provider from environment or default to cerebras
+    llm_provider = os.getenv("GRAPH_LLM_PROVIDER", "cerebras").lower()
+    logger.info(f"   Using LLM provider: {llm_provider.upper()}")
+    
+    try:
+        # Initialize Phase 3 processor
+        processor = GraphBuilderPhase3(llm_provider=llm_provider)
+        
+        # Generate summaries
+        success = processor.generate_summaries()
+        
+        if success:
+            logger.info("")
+            logger.info("✅ Phase 3 completed successfully!")
+            logger.info("� DRIFT search metadata generated and included")
+            logger.info("�📋 Next step: Upload to Neo4j using 3b_save_to_graph_db.py")
+            logger.info("   The graph-data-final.json is now ready for Neo4j import with DRIFT capabilities")
+            return 0
+        else:
+            logger.error("")
+            logger.error("❌ Phase 3 failed")
+            logger.error("   Please check the logs above for details")
+            return 1
+            
+    except Exception as e:
+        logger.error(f"❌ Phase 3 pipeline failed: {e}")
+        import traceback
+        logger.error(traceback.format_exc())
+        return 1
+
+
+if __name__ == "__main__":
+    exit(main())

3_save_to_vector_db.ipynb

@@ -0,0 +1,329 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Save Markdown text into Vector DB"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-1: Config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from my_config import MY_CONFIG"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-2: Read Markdown"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import glob\n",
+    "\n",
+    "pattern = os.path.join(MY_CONFIG.PROCESSED_DATA_DIR, '*.md')\n",
+    "md_file_count = len(glob.glob(pattern, recursive=True)) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loaded 96 documents from 96 files\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_index.core import SimpleDirectoryReader\n",
+    "\n",
+    "reader = SimpleDirectoryReader(input_dir=MY_CONFIG.PROCESSED_DATA_DIR, recursive=False , required_exts=[\".md\"])\n",
+    "documents = reader.load_data()\n",
+    "\n",
+    "print (f\"Loaded {len(documents)} documents from {md_file_count} files\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Doc ID: 20eef2cd-ee21-4dd4-baf6-eda09d5d793b\n",
+      "Text: # Building the open future of AI  We are technology developers,\n",
+      "researchers, industry leaders and advocates who collaborate to advance\n",
+      "safe, responsible AI rooted in open innovation.  ![Conference\n",
+      "Speaker](https://images.prismic.io/ai-alliance/Zy08cq8jQArT0jJI_Imagef\n",
+      "romNotion.jpeg?auto=format%2Ccompress&fit=max&w=3840)  ![Skills &\n",
+      "Education](htt...\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Inspect a sample doc\n",
+    "print (documents[0])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-3: Create Chunks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Created 223 chunks from 96 documents\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_index.core import Document\n",
+    "from llama_index.core.node_parser import SentenceSplitter\n",
+    "\n",
+    "parser = SentenceSplitter(chunk_size=MY_CONFIG.CHUNK_SIZE, chunk_overlap=MY_CONFIG.CHUNK_OVERLAP)\n",
+    "nodes = parser.get_nodes_from_documents(documents)\n",
+    "print(f\"Created {len(nodes)} chunks from {len(documents)} documents\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-4: Setup Embedding Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# If connection to https://huggingface.co/ failed, uncomment the following path\n",
+    "import os\n",
+    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sujee/apps/anaconda3/envs/allycat-6/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "Settings.embed_model = HuggingFaceEmbedding(\n",
+    "    model_name = MY_CONFIG.EMBEDDING_MODEL\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-5: Connect to Milvus"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Connected to Milvus instance:  workspace/rag_website_milvus.db\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Clear up any old data\n",
+    "\n",
+    "from pymilvus import MilvusClient\n",
+    "\n",
+    "milvus_client = MilvusClient(MY_CONFIG.DB_URI)\n",
+    "print (\"✅ Connected to Milvus instance: \", MY_CONFIG.DB_URI )\n",
+    "\n",
+    "# if we already have a collection, clear it first\n",
+    "if milvus_client.has_collection(collection_name = MY_CONFIG.COLLECTION_NAME):\n",
+    "    milvus_client.drop_collection(collection_name = MY_CONFIG.COLLECTION_NAME)\n",
+    "    print ('✅ Cleared collection :', MY_CONFIG.COLLECTION_NAME)\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2025-05-12 23:36:12,218 [DEBUG][_create_connection]: Created new connection using: f81ea0e5320b44f7b5ba8b89f6aa43f7 (async_milvus_client.py:600)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Connected Llama-index to Milvus instance:  workspace/rag_website_milvus.db\n"
+     ]
+    }
+   ],
+   "source": [
+    "# connect llama-index to vector db\n",
+    "\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.vector_stores.milvus import MilvusVectorStore\n",
+    "\n",
+    "vector_store = MilvusVectorStore(\n",
+    "    uri = MY_CONFIG.DB_URI ,\n",
+    "    dim = MY_CONFIG.EMBEDDING_LENGTH , \n",
+    "    collection_name = MY_CONFIG.COLLECTION_NAME,\n",
+    "    overwrite=True\n",
+    ")\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "\n",
+    "print (\"✅ Connected Llama-index to Milvus instance: \", MY_CONFIG.DB_URI )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-6: Save to DB"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 9 μs, sys: 0 ns, total: 9 μs\n",
+      "Wall time: 18.8 μs\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "## We save entire md documents into vector store\n",
+    "\n",
+    "# from llama_index.core import VectorStoreIndex\n",
+    "\n",
+    "# index = VectorStoreIndex.from_documents(\n",
+    "#     documents, storage_context=storage_context\n",
+    "# )\n",
+    "# print (f\"✅ Saved {len(documents)} documents to db: {MY_CONFIG.DB_URI}\" )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully stored 223 chunks in Milvus collection 'pages'\n",
+      "CPU times: user 900 ms, sys: 142 ms, total: 1.04 s\n",
+      "Wall time: 807 ms\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time \n",
+    "\n",
+    "# save chunks into vector db\n",
+    "\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "\n",
+    "index = VectorStoreIndex(\n",
+    "        nodes=nodes,\n",
+    "        storage_context=storage_context,\n",
+    "    )\n",
+    "\n",
+    "print(f\"Successfully stored {len(nodes)} chunks in Milvus collection '{MY_CONFIG.COLLECTION_NAME}'\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "milvus_client.close()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "allycat-6",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

3_save_to_vector_db.py

@@ -0,0 +1,85 @@
+from my_config import MY_CONFIG
+import os
+import glob
+from llama_index.core import SimpleDirectoryReader
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.core import Settings
+from pymilvus import MilvusClient
+from llama_index.core import StorageContext
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.core import VectorStoreIndex
+import logging
+
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+# logger.setLevel(logging.INFO)
+
+# Step-1: Read Markdown files
+pattern = os.path.join(MY_CONFIG.PROCESSED_DATA_DIR, '*.md')
+md_file_count = len(glob.glob(pattern, recursive=True))
+
+reader = SimpleDirectoryReader(input_dir=MY_CONFIG.PROCESSED_DATA_DIR, recursive=False, required_exts=[".md"])
+documents = reader.load_data()
+logger.info (f"Loaded {len(documents)} documents from {md_file_count} files")
+
+# Step-2: Create Chunks
+parser = SentenceSplitter(chunk_size=MY_CONFIG.CHUNK_SIZE, chunk_overlap=MY_CONFIG.CHUNK_OVERLAP)
+nodes = parser.get_nodes_from_documents(documents)
+logger.info (f"Created {len(nodes)} chunks from {len(documents)} documents")
+
+# Step-3: Setup Embedding Model
+os.environ['HF_ENDPOINT'] = MY_CONFIG.HF_ENDPOINT
+
+Settings.embed_model = HuggingFaceEmbedding(
+    model_name = MY_CONFIG.EMBEDDING_MODEL
+)
+
+# Step-4: Create 2 Vector Databases (Vector RAG and Hybrid GraphRAG databases)
+
+databases_to_create = [
+    {
+        "name": "Vector RAG Only",
+        "uri": MY_CONFIG.MILVUS_URI_VECTOR,
+        "description": "For Vector RAG systems"
+    },
+    {
+        "name": "Hybrid GraphRAG", 
+        "uri": MY_CONFIG.MILVUS_URI_HYBRID_GRAPH,
+        "description": "For Hybrid GraphRAG systems"
+    }
+]
+
+for db_config in databases_to_create:
+    logger.info(f"📦 Creating {db_config['name']} database...")
+    
+    # Connect to Milvus for this database
+    milvus_client = MilvusClient(db_config['uri'])
+    logger.info(f"✅ Connected to: {db_config['uri']}")
+
+    if milvus_client.has_collection(collection_name = MY_CONFIG.COLLECTION_NAME):
+        milvus_client.drop_collection(collection_name = MY_CONFIG.COLLECTION_NAME)
+        logger.info(f"✅ Cleared collection: {MY_CONFIG.COLLECTION_NAME}")
+
+    # Connect llama-index to vector db
+    vector_store = MilvusVectorStore(
+        uri = db_config['uri'],
+        dim = MY_CONFIG.EMBEDDING_LENGTH,
+        collection_name = MY_CONFIG.COLLECTION_NAME,
+        overwrite=True
+    )
+    storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+    # Save chunks into vector db
+    index = VectorStoreIndex(
+        nodes=nodes,
+        storage_context=storage_context,
+    )
+
+    logger.info(f"✅ Stored {len(nodes)} chunks in {db_config['name']}")
+    milvus_client.close()
+
+logger.info("🎉 Both databases created!")
+logger.info(f"   • Vector RAG: {MY_CONFIG.MILVUS_URI_VECTOR}")
+logger.info(f"   • Hybrid GraphRAG: {MY_CONFIG.MILVUS_URI_HYBRID_GRAPH}")

3_save_to_vector_db_zilliz.py

@@ -0,0 +1,106 @@
+"""
+Cloud Vector Database Setup
+
+Creates vector database collections on cloud infrastructure.
+Supports both vector search and graph-based retrieval systems.
+"""
+
+from my_config import MY_CONFIG
+import os
+import sys
+from llama_index.core import SimpleDirectoryReader
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.core import Settings
+from pymilvus import MilvusClient
+from llama_index.core import StorageContext
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.core import VectorStoreIndex
+import logging
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Validate cloud database configuration
+if not MY_CONFIG.ZILLIZ_CLUSTER_ENDPOINT:
+    raise ValueError("Cloud endpoint configuration missing")
+if not MY_CONFIG.ZILLIZ_TOKEN:
+    raise ValueError("Cloud authentication token missing")
+
+def main():
+    logger.info("Initializing cloud database connection")
+
+    # Load source documents
+    logger.info("Loading documents")
+    reader = SimpleDirectoryReader(input_dir=MY_CONFIG.PROCESSED_DATA_DIR, recursive=False, required_exts=[".md"])
+    documents = reader.load_data()
+    logger.info(f"Loaded {len(documents)} documents")
+
+    # Process document chunks
+    logger.info("Processing document chunks")
+    parser = SentenceSplitter(chunk_size=MY_CONFIG.CHUNK_SIZE, chunk_overlap=MY_CONFIG.CHUNK_OVERLAP)
+    nodes = parser.get_nodes_from_documents(documents)
+    logger.info(f"Created {len(nodes)} chunks")
+
+    # Initialize embedding model
+    logger.info("Configuring embedding model")
+    os.environ['HF_ENDPOINT'] = MY_CONFIG.HF_ENDPOINT
+
+    Settings.embed_model = HuggingFaceEmbedding(
+        model_name=MY_CONFIG.EMBEDDING_MODEL
+    )
+
+    # Create cloud database collection
+    logger.info("Creating database collection")
+    collection_name = MY_CONFIG.COLLECTION_NAME
+
+    milvus_client = None
+    try:
+        # Connect to cloud database
+        milvus_client = MilvusClient(
+            uri=MY_CONFIG.ZILLIZ_CLUSTER_ENDPOINT,
+            token=MY_CONFIG.ZILLIZ_TOKEN
+        )
+
+        # Remove existing collection if present
+        if milvus_client.has_collection(collection_name=collection_name):
+            milvus_client.drop_collection(collection_name=collection_name)
+
+        # Initialize vector store
+        vector_store = MilvusVectorStore(
+            uri=MY_CONFIG.ZILLIZ_CLUSTER_ENDPOINT,
+            token=MY_CONFIG.ZILLIZ_TOKEN,
+            collection_name=collection_name,
+            dim=MY_CONFIG.EMBEDDING_LENGTH,
+            overwrite=True
+        )
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+        # Store document vectors
+        logger.info(f"Processing {len(nodes)} document chunks")
+        VectorStoreIndex(
+            nodes=nodes,
+            storage_context=storage_context,
+        )
+
+        logger.info(f"Database collection '{collection_name}' created successfully")
+        
+    except Exception as e:
+        logger.error(f"Failed to create collection: {str(e)}")
+        raise
+    finally:
+        if milvus_client:
+            milvus_client.close()
+
+    logger.info("Cloud database setup completed successfully")
+
+if __name__ == "__main__":
+    try:
+        main()
+        sys.exit(0)
+    except KeyboardInterrupt:
+        logger.info("Operation cancelled by user")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"Fatal error: {str(e)}")
+        sys.exit(1)

3b_save_to_graph_db.py

@@ -0,0 +1,1050 @@
+import asyncio
+import json
+import logging
+import os
+import sys
+from typing import Any, Dict, Optional
+from my_config import MY_CONFIG
+from neo4j import GraphDatabase, Driver
+from tqdm import tqdm
+from fastmcp import FastMCP
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+GRAPH_DATA_DIR = MY_CONFIG.GRAPH_DATA_DIR
+GRAPH_DATA_FILE = os.path.join(GRAPH_DATA_DIR, "graph-data-final.json")
+
+class Neo4jConnection:
+    def __init__(self):
+        self.uri = MY_CONFIG.NEO4J_URI
+        self.username = MY_CONFIG.NEO4J_USER
+        self.password = MY_CONFIG.NEO4J_PASSWORD
+        self.database = getattr(MY_CONFIG, "NEO4J_DATABASE", None)
+        if not self.uri:
+            raise ValueError("NEO4J_URI config is required")
+        if not self.username:
+            raise ValueError("NEO4J_USERNAME config is required")
+        if not self.password:
+            raise ValueError("NEO4J_PASSWORD config is required")
+        if not self.database:
+            raise ValueError("NEO4J_DATABASE config is required")
+        self.driver: Optional[Driver] = None
+    
+    async def connect(self):
+        if self.driver is None:
+            try:
+                self.driver = GraphDatabase.driver(
+                    self.uri,
+                    auth=(self.username, self.password)
+                )
+                
+                await asyncio.get_event_loop().run_in_executor(
+                    None, self.driver.verify_connectivity
+                )
+                logger.info("Connected to Neo4j")
+                
+            except Exception as e:
+                logger.error(f"Connection failed: {e}")
+                self.driver = None
+    
+    async def disconnect(self):
+        if self.driver:
+            await asyncio.get_event_loop().run_in_executor(
+                None, self.driver.close
+            )
+            self.driver = None
+    
+    async def execute_query(self, query: str, parameters: Optional[Dict[str, Any]] = None):
+        if not self.driver:
+            raise ConnectionError("Not connected to Neo4j database")
+        
+        def run_query():
+            with self.driver.session(database=self.database) as session:
+                result = session.run(query, parameters or {})
+                records = [record.data() for record in result]
+                summary = result.consume()
+                return records, summary
+        
+        return await asyncio.get_event_loop().run_in_executor(None, run_query)
+
+neo4j_connection = Neo4jConnection()
+
+app = FastMCP("Neo4j Graph Data Upload Server")
+
+@app.tool()
+async def execute_cypher(query: str, parameters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    try:
+        if not neo4j_connection.driver:
+            await neo4j_connection.connect()
+            if not neo4j_connection.driver:
+                return {
+                    "status": "error",
+                    "error": "Unable to connect to Neo4j database",
+                    "details": "Check connection settings and network connectivity"
+                }
+        
+        records, summary = await neo4j_connection.execute_query(query, parameters)
+        
+        return {
+            "status": "success",
+            "query": query,
+            "parameters": parameters or {},
+            "records": records,
+            "record_count": len(records),
+            "execution_time_ms": summary.result_available_after,
+            "summary": {
+                "query_type": summary.query_type,
+                "counters": dict(summary.counters) if summary.counters else {}
+            }
+        }
+        
+    except Exception as e:
+        return {
+            "status": "error",
+            "query": query,
+            "error": str(e)
+        }
+
+
+@app.tool()
+async def get_database_schema() -> Dict[str, Any]:
+    try:
+        if not neo4j_connection.driver:
+            await neo4j_connection.connect()
+            if not neo4j_connection.driver:
+                return {
+                    "status": "error",
+                    "error": "Unable to connect to Neo4j database"
+                }
+        
+        labels_records, _ = await neo4j_connection.execute_query("CALL db.labels()")
+        labels = [record["label"] for record in labels_records]
+        
+        rel_records, _ = await neo4j_connection.execute_query("CALL db.relationshipTypes()")
+        relationships = [record["relationshipType"] for record in rel_records]
+        
+        prop_records, _ = await neo4j_connection.execute_query("CALL db.propertyKeys()")
+        properties = [record["propertyKey"] for record in prop_records]
+
+        try:
+            constraint_records, _ = await neo4j_connection.execute_query("SHOW CONSTRAINTS")
+            constraints = [dict(record) for record in constraint_records]
+        except Exception:
+            constraints = []
+        
+        try:
+            index_records, _ = await neo4j_connection.execute_query("SHOW INDEXES")
+            indexes = [dict(record) for record in index_records]
+        except Exception:
+            indexes = []
+        
+        return {
+            "status": "success",
+            "schema": {
+                "node_labels": labels,
+                "relationship_types": relationships,
+                "property_keys": properties,
+                "constraints": constraints,
+                "indexes": indexes
+            }
+        }
+        
+    except Exception as e:
+        return {
+            "status": "error",
+            "error": str(e)
+        }
+
+
+@app.tool()
+async def get_node_count(label: Optional[str] = None) -> Dict[str, Any]:
+    try:
+        if not neo4j_connection.driver:
+            await neo4j_connection.connect()
+            if not neo4j_connection.driver:
+                return {
+                    "status": "error",
+                    "error": "Unable to connect to Neo4j database"
+                }
+        
+        if label:
+            query = f"MATCH (n:`{label}`) RETURN count(n) as count"
+        else:
+            query = "MATCH (n) RETURN count(n) as count"
+        
+        records, _ = await neo4j_connection.execute_query(query)
+        count = records[0]["count"] if records else 0
+        
+        return {
+            "status": "success",
+            "label": label,
+            "count": count
+        }
+        
+    except Exception as e:
+        return {
+            "status": "error",
+            "error": str(e)
+        }
+
+
+@app.tool()
+async def get_relationship_count(relationship_type: Optional[str] = None) -> Dict[str, Any]:
+    try:
+        if not neo4j_connection.driver:
+            await neo4j_connection.connect()
+            if not neo4j_connection.driver:
+                return {
+                    "status": "error",
+                    "error": "Unable to connect to Neo4j database"
+                }
+        
+        if relationship_type:
+            query = f"MATCH ()-[r:`{relationship_type}`]-() RETURN count(r) as count"
+        else:
+            query = "MATCH ()-[r]-() RETURN count(r) as count"
+        
+        records, _ = await neo4j_connection.execute_query(query)
+        count = records[0]["count"] if records else 0
+        
+        return {
+            "status": "success",
+            "relationship_type": relationship_type,
+            "count": count
+        }
+        
+    except Exception as e:
+        return {
+            "status": "error",
+            "error": str(e)
+        }
+
+
+@app.tool()
+async def health_check() -> Dict[str, Any]:
+    try:
+        if not neo4j_connection.driver:
+            await neo4j_connection.connect()
+        
+        if not neo4j_connection.driver:
+            return {
+                "status": "unhealthy",
+                "reason": "Unable to connect to Neo4j database",
+                "configuration": {
+                    "uri": neo4j_connection.uri,
+                    "database": neo4j_connection.database,
+                    "username": neo4j_connection.username
+                }
+            }
+        
+        # A simple query to test connectivity
+        records, _ = await neo4j_connection.execute_query("RETURN 1 as test")
+        
+        if records and records[0]["test"] == 1:
+            return {
+                "status": "healthy",
+                "database": neo4j_connection.database,
+                "uri": neo4j_connection.uri,
+                "ssl_enabled": neo4j_connection.uri.startswith(('neo4j+s://', 'bolt+s://')),
+                "message": "Neo4j connection is working properly"
+            }
+        else:
+            return {
+                "status": "unhealthy",
+                "reason": "Query execution failed or returned unexpected results"
+            }
+            
+    except Exception as e:
+        return {
+            "status": "unhealthy",
+            "reason": str(e)
+        }
+
+
+async def clear_database_impl() -> Dict[str, Any]:
+    try:
+        if not neo4j_connection.driver:
+            await neo4j_connection.connect()
+            if not neo4j_connection.driver:
+                return {
+                    "status": "error",
+                    "error": "Unable to connect to Neo4j database"
+                }
+        
+        node_count_query = "MATCH (n) RETURN count(n) as count"
+        rel_count_query = "MATCH ()-[r]->() RETURN count(r) as count"
+        
+        node_records, _ = await neo4j_connection.execute_query(node_count_query)
+        rel_records, _ = await neo4j_connection.execute_query(rel_count_query)
+        
+        nodes_before = node_records[0]["count"] if node_records else 0
+        rels_before = rel_records[0]["count"] if rel_records else 0
+        
+        await neo4j_connection.execute_query("MATCH ()-[r]->() DELETE r")
+        await neo4j_connection.execute_query("MATCH (n) DELETE n")
+        
+        print(f"✅ Cleared: {nodes_before} nodes, {rels_before} relationships")
+        
+        return {
+            "status": "success",
+            "message": "Database cleared successfully",
+            "statistics": {
+                "nodes_removed": nodes_before,
+                "relationships_removed": rels_before
+            }
+        }
+        
+    except Exception as e:
+        return {
+            "status": "error",
+            "error": str(e)
+        }
+
+
+@app.tool()
+async def clear_database() -> Dict[str, Any]:
+    return await clear_database_impl()
+
+
+async def upload_graph_data_impl() -> Dict[str, Any]:
+    try:
+        if not neo4j_connection.driver:
+            await neo4j_connection.connect()
+            if not neo4j_connection.driver:
+                return {
+                    "status": "error",
+                    "error": "Unable to connect to Neo4j database"
+                }
+        
+        clear_result = await clear_database_impl()
+        if clear_result["status"] != "success":
+            return clear_result
+        
+        # Check if graph data file exists
+        if not os.path.exists(GRAPH_DATA_FILE):
+            return {
+                "status": "error",
+                "error": f"Graph data file not found: {GRAPH_DATA_FILE}"
+            }
+        
+        with open(GRAPH_DATA_FILE, 'r', encoding='utf-8') as f:
+            graph_data = json.load(f)
+        
+        if not isinstance(graph_data, dict) or 'nodes' not in graph_data:
+            return {
+                "status": "error",
+                "error": "Invalid graph data format. Expected JSON with 'nodes' array"
+            }
+        
+        nodes = graph_data.get('nodes', [])
+        relationships = graph_data.get('relationships', [])
+        communities_data = graph_data.get('communities', {})
+        drift_metadata = graph_data.get('drift_search_metadata', {})
+        global_metadata = graph_data.get('metadata', {})
+        search_optimization = drift_metadata.get('search_optimization', {}) if drift_metadata else {}
+        
+        communities_count = len(drift_metadata.get('community_search_index', {})) if drift_metadata else 0
+        drift_count = 1 if drift_metadata else 0
+        metadata_count = 1 if global_metadata else 0
+        optimization_count = 1 if search_optimization else 0
+        communities_metadata_count = 1 if communities_data else 0
+        drift_config_count = 1 if (drift_metadata and 'configuration' in drift_metadata) else 0
+        community_search_index_count = 1 if (drift_metadata and 'community_search_index' in drift_metadata) else 0
+        search_optimization_object_count = 1 if (drift_metadata and 'search_optimization' in drift_metadata) else 0
+        embeddings_object_count = 1 if (drift_metadata and 'community_search_index' in drift_metadata) else 0
+        embeddings_count = communities_count if (drift_metadata and 'community_search_index' in drift_metadata) else 0
+        
+        total_items = (len(nodes) + len(relationships) + communities_count + drift_count + 
+                      metadata_count + optimization_count + communities_metadata_count + 
+                      drift_config_count + community_search_index_count + 
+                      search_optimization_object_count + embeddings_object_count + embeddings_count)
+        
+        print(f"Processing: {len(nodes)} nodes, {len(relationships)} relationships, {communities_count} communities, {total_items - len(nodes) - len(relationships) - communities_count} metadata")
+        
+        upload_stats = {
+            "nodes_processed": 0,
+            "nodes_created": 0,
+            "relationships_processed": 0,
+            "relationships_created": 0,
+            "communities_processed": 0,
+            "communities_created": 0,
+            "drift_metadata_created": 0,
+            "global_metadata_created": 0,
+            "search_optimization_created": 0,
+            "communities_metadata_created": 0,
+            "drift_config_created": 0,
+            "community_search_index_created": 0,
+            "search_optimization_object_created": 0,
+            "embeddings_object_created": 0,
+            "embeddings_created": 0,
+            "errors": []
+        }
+        
+        with tqdm(total=len(nodes), desc="Nodes", unit="node", ncols=80, leave=False) as pbar:
+            for node in nodes:
+                try:
+                    upload_stats["nodes_processed"] += 1
+                    
+                    node_id = node['id']
+                    labels = node['labels']
+                    properties = node.get('properties', {})
+                    
+                    # Create node with labels
+                    labels_str = ':'.join([f"`{label}`" for label in labels])
+                    query = f"MERGE (n:{labels_str} {{id: $id}}) SET n += $props RETURN n"
+                    
+                    await neo4j_connection.execute_query(query, {
+                        "id": node_id,
+                        "props": properties
+                    })
+                    
+                    upload_stats["nodes_created"] += 1
+                    pbar.update(1)
+                    
+                except Exception as e:
+                    upload_stats["errors"].append(f"Node upload error: {str(e)}")
+                    pbar.update(1)
+        
+        with tqdm(total=len(relationships), desc="Relationships", unit="rel", ncols=80, leave=False) as pbar:
+            for rel in relationships:
+                upload_stats["relationships_processed"] += 1
+
+                start_node = rel['startNode']
+                end_node = rel['endNode'] 
+                rel_type = rel['type']
+                
+                try:
+                    query = f"""
+                    MATCH (a {{id: $start_node}})
+                    MATCH (b {{id: $end_node}})
+                    CREATE (a)-[r:`{rel_type}`]->(b)
+                    SET r += $props
+                    RETURN r
+                    """
+                    
+                    await neo4j_connection.execute_query(query, {
+                        "start_node": start_node,
+                        "end_node": end_node,
+                        "props": properties
+                    })
+                    
+                    upload_stats["relationships_created"] += 1
+                    pbar.update(1)
+                    
+                except Exception as e:
+                    error_msg = f"Relationship upload error for rel {rel}: {str(e)}"
+                    logger.error(error_msg)
+                    upload_stats["errors"].append(error_msg)
+                    pbar.update(1)
+        
+        if drift_metadata and 'community_search_index' in drift_metadata:
+            community_index = drift_metadata['community_search_index']
+            
+            with tqdm(total=len(community_index), desc="Communities", unit="comm", ncols=80, leave=False) as pbar:
+                for comm_id, comm_data in community_index.items():
+                    try:
+                        upload_stats["communities_processed"] += 1
+                        
+                        embeddings = comm_data.get('embeddings', {})
+                        summary_embedding = embeddings.get('summary_embedding', [])
+                        hyde_embeddings = embeddings.get('hyde_embeddings', [])
+                        
+                        follow_up_templates_json = json.dumps(comm_data.get('follow_up_templates', {}))
+                        hyde_embeddings_json = json.dumps(hyde_embeddings)
+                        
+                        # Get statistics
+                        stats = comm_data.get('statistics', {})
+                        
+                        # Community properties with documented attributes from JSON
+                        community_props = {
+                            "id": comm_id,
+                            "summary": comm_data.get('summary', ''),
+                            "key_entities": comm_data.get('key_entities', []),
+                            "member_count": stats.get('member_count', 0),
+                            "member_ids": stats.get('member_ids', []),
+                            "internal_edges": stats.get('internal_edges', 0),
+                            "density": stats.get('density', 0.0),
+                            "avg_degree": stats.get('avg_degree', 0.0),
+                            "follow_up_templates": follow_up_templates_json,
+                            "hyde_embeddings": hyde_embeddings_json
+                        }
+                        
+                        # Add summary embedding as List<Float> if available
+                        if summary_embedding and isinstance(summary_embedding, list):
+                            community_props["summary_embedding"] = summary_embedding
+                            community_props["embedding_dimensions"] = len(summary_embedding)
+                        
+                        # Create Community node
+                        query = """
+                        MERGE (c:Community {id: $id})
+                        SET c += $props
+                        RETURN c
+                        """
+                        
+                        await neo4j_connection.execute_query(query, {
+                            "id": comm_id,
+                            "props": community_props
+                        })
+                        
+                        upload_stats["communities_created"] += 1
+                        pbar.update(1)
+                        
+                    except Exception as e:
+                        error_msg = f"Community upload error for {comm_id}: {str(e)}"
+                        logger.error(error_msg)
+                        upload_stats["errors"].append(error_msg)
+                        pbar.update(1)
+        
+        if drift_metadata:
+            try:
+                query_routing_config_json = json.dumps(drift_metadata.get('query_routing_config', {}))
+                performance_monitoring_json = json.dumps(drift_metadata.get('performance_monitoring', {}))
+                configuration_json = json.dumps(drift_metadata.get('configuration', {}))
+                community_search_index_json = json.dumps(drift_metadata.get('community_search_index', {}))
+                search_optimization_json = json.dumps(drift_metadata.get('search_optimization', {}))
+
+                # Build a compact embeddings object (per-community) to store on the DRIFT node
+                embeddings_per_community = {}
+                for _comm_id, _comm_data in drift_metadata.get('community_search_index', {}).items():
+                    emb = _comm_data.get('embeddings')
+                    if emb:
+                        # Only keep summary and hyde to limit size
+                        embeddings_per_community[_comm_id] = {
+                            'summary_embedding': emb.get('summary_embedding'),
+                            'hyde_embeddings': emb.get('hyde_embeddings')
+                        }
+
+                embeddings_json = json.dumps(embeddings_per_community)
+
+                drift_props = {
+                    "version": drift_metadata.get('version', '1.0'),
+                    "generated_timestamp": drift_metadata.get('generated_timestamp', ''),
+                    "query_routing_config": query_routing_config_json,
+                    "performance_monitoring": performance_monitoring_json,
+                    "configuration": configuration_json,
+                    # Nested objects stored as JSON strings for direct inspection
+                    "community_search_index": community_search_index_json,
+                    "search_optimization": search_optimization_json,
+                    "embeddings": embeddings_json,
+                    "total_communities": len(drift_metadata.get('community_search_index', {}))
+                }
+                
+                # Create single DRIFT metadata node
+                query = """
+                MERGE (d:DriftMetadata {version: $version})
+                SET d += $props
+                RETURN d
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "version": drift_metadata.get('version', '1.0'),
+                    "props": drift_props
+                })
+                
+                upload_stats["drift_metadata_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"DRIFT metadata upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if global_metadata:
+            try:
+                # Convert nested objects to JSON strings for Neo4j compatibility
+                recovery_stats_json = json.dumps(global_metadata.get('recovery_stats', {}))
+                member_extraction_stats_json = json.dumps(global_metadata.get('member_extraction_stats', {}))
+                community_detection_json = json.dumps(global_metadata.get('community_detection', {}))
+                
+                metadata_props = {
+                    "node_count": global_metadata.get('node_count', 0),
+                    "relationship_count": global_metadata.get('relationship_count', 0),
+                    "generated_at": global_metadata.get('generated_at', ''),
+                    "generator": global_metadata.get('generator', ''),
+                    "llm_provider": global_metadata.get('llm_provider', ''),
+                    "model": global_metadata.get('model', ''),
+                    "format_version": global_metadata.get('format_version', ''),
+                    "last_updated": global_metadata.get('last_updated', ''),
+                    "phase": global_metadata.get('phase', ''),
+                    "entity_count": global_metadata.get('entity_count', global_metadata.get('node_count', 0)),
+                    "community_count": global_metadata.get('community_count', 0),
+                    "total_node_count": global_metadata.get('total_node_count', global_metadata.get('node_count', 0)),
+                    "total_relationship_count": global_metadata.get('total_relationship_count', global_metadata.get('relationship_count', 0)),
+                    
+                    # Complex nested objects as JSON strings
+                    "recovery_stats": recovery_stats_json,
+                    "member_extraction_stats": member_extraction_stats_json,
+                    "community_detection": community_detection_json
+                }
+                
+                # Create Global Metadata node
+                query = """
+                MERGE (m:GraphMetadata {generator: $generator})
+                SET m += $props
+                RETURN m
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "generator": global_metadata.get('generator', 'unknown'),
+                    "props": metadata_props
+                })
+                
+                upload_stats["global_metadata_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"Global metadata upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if search_optimization:
+            try:
+                optimization_props = {
+                    "total_communities": search_optimization.get('total_communities', 0),
+                    "avg_community_size": search_optimization.get('avg_community_size', 0.0),
+                    "graph_density": search_optimization.get('graph_density', 0.0),
+                    "total_nodes": search_optimization.get('total_nodes', 0),
+                    "total_edges": search_optimization.get('total_edges', 0),
+                    "max_primer_communities": search_optimization.get('max_primer_communities', 0)
+                }
+                
+                query = """
+                MERGE (s:SearchOptimization {id: 'global'})
+                SET s += $props
+                RETURN s
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "props": optimization_props
+                })
+                
+                upload_stats["search_optimization_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"Search optimization upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if communities_data:
+            try:
+                communities_props = {
+                    "algorithm": communities_data.get('algorithm', ''),
+                    "total_communities": communities_data.get('total_communities', 0),
+                    "modularity_score": communities_data.get('modularity_score', 0.0),
+                    "summaries": json.dumps(communities_data.get('summaries', {})),
+                    "statistics": json.dumps(communities_data.get('statistics', {}))
+                }
+                
+                query = """
+                MERGE (cm:CommunitiesMetadata {algorithm: $algorithm})
+                SET cm += $props
+                RETURN cm
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "algorithm": communities_data.get('algorithm', 'unknown'),
+                    "props": communities_props
+                })
+                
+                upload_stats["communities_metadata_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"Communities metadata upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if drift_metadata and 'configuration' in drift_metadata:
+            try:
+                config = drift_metadata['configuration']
+                
+                drift_config_props = {
+                    "max_iterations": config.get('max_iterations', 0),
+                    "confidence_threshold": config.get('confidence_threshold', 0.0),
+                    "top_k_communities": config.get('top_k_communities', 0),
+                    "hyde_expansion_count": config.get('hyde_expansion_count', 0),
+                    "termination_criteria": config.get('termination_criteria', ''),
+                    "version": drift_metadata.get('version', '1.0'),
+                    "generated_timestamp": drift_metadata.get('generated_timestamp', '')
+                }
+                
+                query = """
+                MERGE (dc:DriftConfiguration {version: $version})
+                SET dc += $props
+                RETURN dc
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "version": drift_metadata.get('version', '1.0'),
+                    "props": drift_config_props
+                })
+                
+                upload_stats["drift_config_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"DRIFT Configuration upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if drift_metadata and 'community_search_index' in drift_metadata:
+            try:
+                community_search_index = drift_metadata['community_search_index']
+                
+                search_index_props = {
+                    "version": drift_metadata.get('version', '1.0'),
+                    "total_communities": len(community_search_index),
+                    "community_data": json.dumps(community_search_index),
+                    "generated_timestamp": drift_metadata.get('generated_timestamp', ''),
+                    "index_type": "community_search"
+                }
+                
+                query = """
+                MERGE (csi:CommunitySearchIndex {version: $version})
+                SET csi += $props
+                RETURN csi
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "version": drift_metadata.get('version', '1.0'),
+                    "props": search_index_props
+                })
+                
+                upload_stats["community_search_index_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"Community Search Index upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if drift_metadata and 'search_optimization' in drift_metadata:
+            try:
+                search_opt_data = drift_metadata['search_optimization']
+                
+                search_opt_props = {
+                    "total_communities": search_opt_data.get('total_communities', 0),
+                    "avg_community_size": search_opt_data.get('avg_community_size', 0.0),
+                    "graph_density": search_opt_data.get('graph_density', 0.0),
+                    "total_nodes": search_opt_data.get('total_nodes', 0),
+                    "total_edges": search_opt_data.get('total_edges', 0),
+                    "max_primer_communities": search_opt_data.get('max_primer_communities', 0),
+                    "optimization_version": drift_metadata.get('version', '1.0')
+                }
+                
+                query = """
+                MERGE (so:SearchOptimizationObject {optimization_version: $version})
+                SET so += $props
+                RETURN so
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "version": drift_metadata.get('version', '1.0'),
+                    "props": search_opt_props
+                })
+                
+                upload_stats["search_optimization_object_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"Search Optimization object upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if drift_metadata and 'community_search_index' in drift_metadata:
+            try:
+                community_index = drift_metadata['community_search_index']
+                
+                total_embeddings = 0
+                total_dimensions = 0
+                embedding_communities = []
+                
+                for comm_id, comm_data in community_index.items():
+                    embeddings_data = comm_data.get('embeddings', {})
+                    if embeddings_data:
+                        total_embeddings += 1
+                        if embeddings_data.get('summary_embedding'):
+                            total_dimensions = len(embeddings_data.get('summary_embedding', []))
+                        embedding_communities.append(comm_id)
+                
+                # Create embeddings object properties
+                embeddings_obj_props = {
+                    "total_embeddings": total_embeddings,
+                    "embedding_dimensions": total_dimensions,
+                    "embedding_computation": "computed via text-embedding-ada-002",
+                    "communities_with_embeddings": embedding_communities,
+                    "embedding_type": "community_summaries",
+                    "embeddings_version": drift_metadata.get('version', '1.0')
+                }
+                
+                query = """
+                MERGE (eo:EmbeddingsObject {embeddings_version: $version})
+                SET eo += $props
+                RETURN eo
+                """
+                
+                await neo4j_connection.execute_query(query, {
+                    "version": drift_metadata.get('version', '1.0'),
+                    "props": embeddings_obj_props
+                })
+                
+                upload_stats["embeddings_object_created"] = 1
+                
+            except Exception as e:
+                error_msg = f"Embeddings object upload error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        if drift_metadata and 'community_search_index' in drift_metadata:
+            community_index = drift_metadata['community_search_index']
+            
+            for comm_id, comm_data in community_index.items():
+                try:
+                    embeddings_data = comm_data.get('embeddings', {})
+                    if embeddings_data:
+                        embeddings_props = {
+                            "community_id": comm_id,
+                            "summary_embedding": embeddings_data.get('summary_embedding', []),
+                            "hyde_embeddings": json.dumps(embeddings_data.get('hyde_embeddings', [])),
+                            "embedding_dimensions": len(embeddings_data.get('summary_embedding', [])),
+                            "embedding_computation": embeddings_data.get('embedding_computation', 'computed')
+                        }
+                        
+                        query = """
+                        MERGE (e:Embeddings {community_id: $community_id})
+                        SET e += $props
+                        RETURN e
+                        """
+                        
+                        await neo4j_connection.execute_query(query, {
+                            "community_id": comm_id,
+                            "props": embeddings_props
+                        })
+                        
+                        upload_stats["embeddings_created"] += 1
+                        
+                except Exception as e:
+                    error_msg = f"Embeddings upload error for {comm_id}: {str(e)}"
+                    logger.error(error_msg)
+                    upload_stats["errors"].append(error_msg)
+        
+        if communities_count > 0:
+            try:
+                # Connect entities to their communities based on community_id property
+                community_rel_query = """
+                MATCH (n) WHERE n.community_id IS NOT NULL
+                MATCH (c:Community {id: n.community_id})
+                MERGE (n)-[:BELONGS_TO_COMMUNITY]->(c)
+                """
+                await neo4j_connection.execute_query(community_rel_query, {})
+                
+            except Exception as e:
+                error_msg = f"Community relationship creation error: {str(e)}"
+                logger.error(error_msg)
+                upload_stats["errors"].append(error_msg)
+        
+        # Calculate success percentage for all components
+        nodes_success_rate = (upload_stats["nodes_created"] / len(nodes) * 100) if nodes else 100
+        rels_success_rate = (upload_stats["relationships_created"] / len(relationships) * 100) if relationships else 100
+        communities_success_rate = (upload_stats["communities_created"] / communities_count * 100) if communities_count else 100
+        drift_success_rate = (upload_stats["drift_metadata_created"] / drift_count * 100) if drift_count else 100
+        
+        embedding_dimensions = 0
+        if drift_metadata and 'community_search_index' in drift_metadata:
+            for comm_data in drift_metadata['community_search_index'].values():
+                embeddings = comm_data.get('embeddings', {})
+                summary_embedding = embeddings.get('summary_embedding', [])
+                if summary_embedding:
+                    embedding_dimensions = len(summary_embedding)
+                    break
+        
+        total_created = (upload_stats["nodes_created"] + upload_stats["relationships_created"] + 
+                        upload_stats["communities_created"] + upload_stats["drift_metadata_created"] +
+                        upload_stats["global_metadata_created"] + upload_stats["search_optimization_created"] +
+                        upload_stats["communities_metadata_created"] + upload_stats["drift_config_created"] +
+                        upload_stats["community_search_index_created"] + upload_stats["search_optimization_object_created"] +
+                        upload_stats["embeddings_object_created"] + upload_stats["embeddings_created"])
+        overall_success_rate = (total_created / total_items * 100) if total_items else 100
+        
+        result = {
+            "status": "success",
+            "message": "Graph data upload completed successfully",
+            "statistics": upload_stats,
+            "success_rates": {
+                "nodes": f"{nodes_success_rate:.1f}%",
+                "relationships": f"{rels_success_rate:.1f}%",
+                "communities": f"{communities_success_rate:.1f}%",
+                "drift_metadata": f"{drift_success_rate:.1f}%",
+                "global_metadata": f"{100.0 if upload_stats['global_metadata_created'] > 0 else 0:.1f}%",
+                "search_optimization": f"{100.0 if upload_stats['search_optimization_created'] > 0 else 0:.1f}%",
+                "communities_metadata": f"{100.0 if upload_stats['communities_metadata_created'] > 0 else 0:.1f}%",
+                "drift_config": f"{100.0 if upload_stats['drift_config_created'] > 0 else 0:.1f}%",
+                "community_search_index": f"{100.0 if upload_stats['community_search_index_created'] > 0 else 0:.1f}%",
+                "search_optimization_object": f"{100.0 if upload_stats['search_optimization_object_created'] > 0 else 0:.1f}%",
+                "embeddings_object": f"{100.0 if upload_stats['embeddings_object_created'] > 0 else 0:.1f}%",
+                "embeddings": f"{(upload_stats['embeddings_created']/communities_count*100) if communities_count > 0 else 0:.1f}%",
+                "overall": f"{overall_success_rate:.1f}%"
+            },
+            "source_file": GRAPH_DATA_FILE,
+            "architecture_summary": {
+                "nodes": f"{upload_stats['nodes_created']}/{len(nodes)}",
+                "relationships": f"{upload_stats['relationships_created']}/{len(relationships)}",
+                "communities": f"{upload_stats['communities_created']}/{communities_count}",
+                "drift_metadata": f"{upload_stats['drift_metadata_created']}/{drift_count}",
+                "global_metadata": f"{upload_stats['global_metadata_created']}/1",
+                "search_optimization": f"{upload_stats['search_optimization_created']}/1",
+                "embeddings_stored": communities_count > 0,
+                "vector_dimensions": embedding_dimensions,
+                "complete_metadata_coverage": upload_stats['global_metadata_created'] > 0 and upload_stats['search_optimization_created'] > 0
+            }
+        }
+        
+        # Print concise upload summary
+        print(f"\n✅ Upload completed: {total_created}/{total_items} items ({overall_success_rate:.1f}%)")
+        
+        # Show all node types created
+        total_entity_nodes = upload_stats['nodes_created']
+        total_metadata_nodes = (upload_stats['drift_metadata_created'] + 
+                               upload_stats['global_metadata_created'] + 
+                               upload_stats['search_optimization_created'] +
+                               upload_stats['communities_metadata_created'] + 
+                               upload_stats['drift_config_created'] +
+                               upload_stats['community_search_index_created'] + 
+                               upload_stats['search_optimization_object_created'] +
+                               upload_stats['embeddings_object_created'])
+        
+        print(f"   Entity Nodes: {total_entity_nodes}, Community Nodes: {upload_stats['communities_created']}, Metadata Nodes: {total_metadata_nodes}, Embedding Nodes: {upload_stats['embeddings_created']}")
+        print(f"   Relationships: {upload_stats['relationships_created']}")
+        
+        if upload_stats['errors']:
+            print(f"   ⚠️  {len(upload_stats['errors'])} errors encountered")
+        
+        return result
+        
+    except Exception as e:
+        logger.error(f"Graph data upload failed: {str(e)}")
+        return {
+            "status": "error",
+            "error": str(e)
+        }
+
+
+@app.tool()
+async def upload_graph_data() -> Dict[str, Any]:
+    return await upload_graph_data_impl()
+
+
+@app.tool()
+async def check_graph_data_file() -> Dict[str, Any]:
+    try:
+        if not os.path.exists(GRAPH_DATA_FILE):
+            return {
+                "status": "not_found",
+                "path": GRAPH_DATA_FILE,
+                "message": "Graph data file does not exist"
+            }
+        
+        # Get file stats
+        file_stats = os.stat(GRAPH_DATA_FILE)
+        file_size = file_stats.st_size
+        
+        # Try to parse the JSON to validate format
+        try:
+            with open(GRAPH_DATA_FILE, 'r', encoding='utf-8') as f:
+                graph_data = json.load(f)
+            
+            nodes_count = len(graph_data.get('nodes', []))
+            relationships_count = len(graph_data.get('relationships', []))
+            
+            return {
+                "status": "found",
+                "path": GRAPH_DATA_FILE,
+                "file_size_bytes": file_size,
+                "nodes_count": nodes_count,
+                "relationships_count": relationships_count,
+                "valid_json": True
+            }
+            
+        except json.JSONDecodeError as e:
+            return {
+                "status": "invalid",
+                "path": GRAPH_DATA_FILE,
+                "file_size_bytes": file_size,
+                "valid_json": False,
+                "json_error": str(e)
+            }
+        
+    except Exception as e:
+        return {
+            "status": "error",
+            "error": str(e),
+            "error_type": type(e).__name__
+        }
+
+
+@app.tool()
+async def get_connection_info() -> Dict[str, Any]:
+    try:
+        # Always return configuration info even if not connected
+        deployment_type = "Self-hosted"
+        if "databases.neo4j.io" in neo4j_connection.uri:
+            deployment_type = "Neo4j Aura"
+        elif "sandbox" in neo4j_connection.uri:
+            deployment_type = "Neo4j Sandbox"
+        elif any(cloud in neo4j_connection.uri for cloud in ["aws", "gcp", "azure"]):
+            deployment_type = "Enterprise Cloud"
+        
+        connection_info = {
+            "status": "success",
+            "connection": {
+                "uri": neo4j_connection.uri,
+                "database": neo4j_connection.database,
+                "username": neo4j_connection.username,
+                "deployment_type": deployment_type,
+                "ssl_enabled": neo4j_connection.uri.startswith(('neo4j+s://', 'bolt+s://')),
+                "connected": neo4j_connection.driver is not None
+            },
+            "capabilities": {
+                "cypher_queries": True,
+                "schema_inspection": True,
+                "bulk_operations": True,
+                "graph_algorithms": "unknown",
+                "multi_database": "unknown"
+            }
+        }
+        
+        if neo4j_connection.driver:
+            try:
+                server_info_records, _ = await neo4j_connection.execute_query(
+                    "CALL dbms.components() YIELD name, versions, edition"
+                )
+                connection_info["server_info"] = server_info_records[0] if server_info_records else {}
+            except Exception:
+                connection_info["server_info"] = {}
+        
+        return connection_info
+        
+    except Exception as e:
+        logger.error(f"Connection info retrieval failed: {str(e)}")
+        return {
+            "status": "error",
+            "error": str(e),
+            "error_type": type(e).__name__
+        }
+
+
+if __name__ == "__main__":
+    import sys
+    try:
+        asyncio.run(neo4j_connection.connect())
+        print(f"Looking for graph data at: {GRAPH_DATA_FILE}")
+        print(f"File exists: {os.path.exists(GRAPH_DATA_FILE)}")
+        
+        result = asyncio.run(upload_graph_data_impl())
+        print(f"Upload result: {result.get('status', 'unknown')}")
+        
+        if result.get('status') == 'error':
+            print(f"❌ Error details: {result.get('error', 'Unknown error')}")
+            if 'error_type' in result:
+                print(f"Error type: {result['error_type']}")
+        
+    except ValueError as e:
+        logger.error(f"Configuration Error: {e}")
+        sys.exit(1)
+    except Exception as e:
+        logger.warning(f"Connection Warning: {e}")

4_query copy.py

@@ -0,0 +1,173 @@
+import os
+from my_config import MY_CONFIG
+
+# If connection to https://huggingface.co/ failed, uncomment the following path
+os.environ['HF_ENDPOINT'] = MY_CONFIG.HF_ENDPOINT
+
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.core import Settings
+from llama_index.core import VectorStoreIndex, StorageContext
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.core import VectorStoreIndex
+from dotenv import load_dotenv
+from llama_index.llms.litellm import LiteLLM
+import query_utils
+import time
+import logging
+import json
+
+logging.basicConfig(
+    level=logging.INFO, 
+    format='%(asctime)s - %(levelname)s - %(message)s', 
+    handlers=[
+        logging.FileHandler('logs/query/query_log.txt', mode='a'),  # Save to file
+        logging.StreamHandler()  # Also show in console
+    ],
+    force=True
+)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def run_query(query: str):
+    global query_engine
+    logger.info (f"-----------------------------------")
+    start_time = time.time()
+    query = query_utils.tweak_query(query, MY_CONFIG.LLM_MODEL)
+    logger.info (f"\nProcessing Query:\n{query}")
+    
+    # Get initial vector response
+    vector_response = query_engine.query(query)
+    vector_text = str(vector_response).strip()
+    
+    # Structured prompt 
+    structured_prompt = f"""Please provide a comprehensive, well-structured answer using the provided document information.
+
+Question: {query}
+
+Document Information:
+{vector_text}
+
+Instructions:
+1. Provide accurate, factual information based on the documents
+2. Structure your response clearly with proper formatting
+3. Be comprehensive yet concise
+4. Highlight key relationships and important details when relevant
+5. Use bullet points or sections when appropriate for clarity
+
+Please provide your answer:"""
+    
+    # Use structured prompt for final synthesis
+    res = query_engine.query(structured_prompt)
+    
+    end_time = time.time()
+    total_time = end_time - start_time
+    logger.info ( "-------"
+                 + f"\nResponse:\n{res}" 
+                 + f"\n\n⏱️ Total time: {total_time:.1f} seconds"
+                 + f"\n\nResponse Metadata:\n{json.dumps(res.metadata, indent=2)}" 
+                #  + f"\nSource Nodes: {[node.node_id for node in res.source_nodes]}"
+                 )
+    logger.info (f"-----------------------------------")
+    
+    # Save response and metadata to files
+    _save_query_files(query, res, total_time)
+    
+    return res
+
+def _save_query_files(query: str, response, total_time: float):
+    """Save query response and metadata to files."""
+    import time
+    timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
+    
+    try:
+        # Save response to file
+        with open('logs/query/query_responses.txt', 'a', encoding='utf-8') as f:
+            f.write(f"\n{'='*80}\n")
+            f.write(f"QUERY [{timestamp}]: {query}\n")
+            f.write(f"{'='*80}\n")
+            f.write(f"RESPONSE: {response}\n")
+            f.write(f"TIME: {total_time:.1f} seconds\n")
+            f.write(f"{'='*80}\n\n")
+        
+        # Save metadata to file
+        with open('logs/query/query_metadata.txt', 'a', encoding='utf-8') as f:
+            f.write(f"\n{'='*80}\n")
+            f.write(f"METADATA [{timestamp}]: {query}\n")
+            f.write(f"{'='*80}\n")
+            f.write(f"TIME: {total_time:.1f} seconds\n")
+            f.write(json.dumps(response.metadata, indent=2, default=str))
+            f.write(f"\n{'='*80}\n\n")
+            
+        logger.info(f"Saved response and metadata for query: {query[:50]}...")
+    except Exception as e:
+        logger.error(f"Failed to save query files: {e}")
+
+## ======= end : run_query =======
+
+## load env config
+load_dotenv()
+
+# Setup embeddings
+Settings.embed_model = HuggingFaceEmbedding(
+    model_name = MY_CONFIG.EMBEDDING_MODEL
+)
+logger.info (f"✅ Using embedding model: {MY_CONFIG.EMBEDDING_MODEL}")
+
+# Connect to Vector RAG only database
+vector_store = MilvusVectorStore(
+    uri = MY_CONFIG.MILVUS_URI_VECTOR,  # Use dedicated Vector-only database
+    dim = MY_CONFIG.EMBEDDING_LENGTH,
+    collection_name = MY_CONFIG.COLLECTION_NAME, 
+    overwrite=False  # so we load the index from db
+)
+storage_context = StorageContext.from_defaults(vector_store=vector_store)
+logger.info (f"✅ Connected to Vector-only Milvus instance: {MY_CONFIG.MILVUS_URI_VECTOR}")
+
+# Load Document Index from DB
+
+index = VectorStoreIndex.from_vector_store(
+    vector_store=vector_store, storage_context=storage_context)
+logger.info (f"✅ Loaded Vector-only index from: {MY_CONFIG.MILVUS_URI_VECTOR}")
+
+# Setup LLM
+logger.info (f"✅ Using LLM model : {MY_CONFIG.LLM_MODEL}")
+Settings.llm = LiteLLM (
+        model=MY_CONFIG.LLM_MODEL,
+    )
+ 
+query_engine = index.as_query_engine()
+
+# Sample queries
+queries = [
+    # "What is AI Alliance?",
+    # "What are the main focus areas of AI Alliance?",
+    # "What are some ai alliance projects?",
+    # "What are the upcoming events?", 
+    # "How do I join the AI Alliance?",
+    # "When was the moon landing?",
+]
+
+for query in queries:
+    run_query(query)
+
+logger.info (f"-----------------------------------")
+
+while True:
+    # Get user input
+    user_query = input("\nEnter your question (or 'q' to exit): ")
+    
+    # Check if user wants to quit
+    if user_query.lower() in ['quit', 'exit', 'q']:
+        logger.info ("Goodbye!")
+        break
+    
+    # Process the query
+    if user_query.strip() == "":
+        continue
+    
+    try:
+        run_query(user_query)
+    except Exception as e:
+        logger.error(f"Error processing query: {e}")
+        print(f"Error processing query: {e}")

4_query.ipynb

@@ -0,0 +1,398 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# RAG on HTML documents\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-1: Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from my_config import MY_CONFIG"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-2: Setup Embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# If connection to https://huggingface.co/ failed, uncomment the following path\n",
+    "import os\n",
+    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sujee/my-stuff/projects/ai-alliance/allycat-1/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "Settings.embed_model = HuggingFaceEmbedding(\n",
+    "    model_name = MY_CONFIG.EMBEDDING_MODEL\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-3: Connect to Milvus"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sujee/my-stuff/projects/ai-alliance/allycat-1/.venv/lib/python3.11/site-packages/milvus_lite/__init__.py:15: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n",
+      "  from pkg_resources import DistributionNotFound, get_distribution\n",
+      "2025-07-14 00:23:38,214 [DEBUG][_create_connection]: Created new connection using: async-workspace/rag_website_milvus.db (async_milvus_client.py:599)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Connected to Milvus instance:  workspace/rag_website_milvus.db\n"
+     ]
+    }
+   ],
+   "source": [
+    "# connect to vector db\n",
+    "from llama_index.core import VectorStoreIndex, StorageContext\n",
+    "from llama_index.vector_stores.milvus import MilvusVectorStore\n",
+    "\n",
+    "vector_store = MilvusVectorStore(\n",
+    "    uri = MY_CONFIG.DB_URI ,\n",
+    "    dim = MY_CONFIG.EMBEDDING_LENGTH , \n",
+    "    collection_name = MY_CONFIG.COLLECTION_NAME,\n",
+    "    overwrite=False  # so we load the index from db\n",
+    ")\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "\n",
+    "print (\"✅ Connected to Milvus instance: \", MY_CONFIG.DB_URI )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-4: Load Document Index from DB"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Loaded index from vector db: workspace/rag_website_milvus.db\n",
+      "CPU times: user 109 ms, sys: 16.8 ms, total: 126 ms\n",
+      "Wall time: 123 ms\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "\n",
+    "index = VectorStoreIndex.from_vector_store(\n",
+    "    vector_store=vector_store, storage_context=storage_context)\n",
+    "\n",
+    "print (\"✅ Loaded index from vector db:\", MY_CONFIG.DB_URI )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-5: Setup LLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Using LLM model : ollama/gemma3:1b\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_index.llms.litellm import LiteLLM\n",
+    "\n",
+    "# Setup LLM\n",
+    "print (f\"✅ Using LLM model : {MY_CONFIG.LLM_MODEL}\")\n",
+    "Settings.llm = LiteLLM (\n",
+    "        model=MY_CONFIG.LLM_MODEL,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step-6: Query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The AI Alliance is an international community of researchers, developers, and organizational leaders committed to fostering open innovation across the AI technology landscape to accelerate progress, improve safety, security, diversity and economic competitiveness in AI.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import query_utils\n",
+    "\n",
+    "query_engine = index.as_query_engine()\n",
+    "query = query_utils.tweak_query('What is AI Alliance?', MY_CONFIG.LLM_MODEL)\n",
+    "res = query_engine.query(query)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The AI Alliance is focused on fostering an open community and enabling developers and researchers to accelerate responsible innovation in AI while ensuring scientific rigor, trust, safety, security, diversity and economic competitiveness.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "query = query_utils.tweak_query('What are the main focus areas of AI Alliance?', MY_CONFIG.LLM_MODEL)\n",
+    "res = query_engine.query(query)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Based on the provided text, here are some of the AI Alliance projects mentioned:\n",
+      "\n",
+      "*   FPT Software\n",
+      "*   Hebrew University of Jerusalem\n",
+      "*   Hugging Face\n",
+      "*   IBM\n",
+      "*   Abdus Salam International Centre for Theoretical Physics (ICTP)\n",
+      "*   Imperial College London\n",
+      "*   Indian Institute of Technology Bombay\n",
+      "*   Institute for Computer Science, Artificial Intelligence\n",
+      "*   Intel\n",
+      "*   Keio University\n",
+      "*   LangChain\n",
+      "*   LlamaIndex\n",
+      "*   Linux Foundation\n",
+      "*   Mass Open Cloud Alliance, operated by Boston University and Harvard\n",
+      "*   Meta\n",
+      "*   Mohamed bin Zayed University of Artificial Intelligence\n",
+      "*   MLCommons\n",
+      "*   National Aeronautics and Space Administration\n",
+      "*   National Science Foundation\n",
+      "*   New York University\n",
+      "*   NumFOCUS\n",
+      "*   OpenTeams\n",
+      "*   Oracle\n",
+      "*   Partnership on AI\n",
+      "*   Quansight\n",
+      "*   Red Hat\n",
+      "*   Rensselaer Polytechnic Institute\n",
+      "*   Roadzen\n",
+      "*   Sakana AI\n",
+      "*   SB Intuitions\n",
+      "*   ServiceNow\n",
+      "*   Silo AI\n",
+      "*   Simons Foundation\n",
+      "*   Sony Group\n",
+      "*   Stability AI\n",
+      "*   Together AI\n",
+      "*   TU Munich\n",
+      "*   UC Berkeley College of Computing, Data Science, and Society\n",
+      "*   University of Illinois Urbana-Champaign\n",
+      "*   The University of Notre Dame\n",
+      "*   The University of Texas at Austin\n",
+      "*   The University of Tokyo\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "query = query_utils.tweak_query('What are some ai alliance projects?', MY_CONFIG.LLM_MODEL)\n",
+    "res = query_engine.query(query)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "On August 8th, The AI Alliance hosted Open Source AI Demo Night in San Francisco.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "query = query_utils.tweak_query('Where was the demo night held?', MY_CONFIG.LLM_MODEL)\n",
+    "res = query_engine.query(query)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The AI Alliance is focused on developing and sharing foundational models for science.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "query = query_utils.tweak_query('What is the AI Alliance doing in the area of material science?', MY_CONFIG.LLM_MODEL)\n",
+    "res = query_engine.query(query)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "By submitting this form, you agree that the AI Alliance will collect and process the personal information you provide to keep you informed about AI Alliance initiatives and enable your involvement in AI Alliance activities. Additionally, you agree that the AI Alliance may share the personal information you provide with its member organizations so that they may communicate with you about AI Alliance initiatives and your involvement in AI Alliance activities.\n",
+      "\n",
+      "You may withdraw your consent for the processing of your personal information by the AI Alliance. Please contact us to request a permanent deletion.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "query = query_utils.tweak_query('How do I join the AI Alliance?', MY_CONFIG.LLM_MODEL)\n",
+    "res = query_engine.query(query)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The context does not provide information about the moon landing.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "query = query_utils.tweak_query('When was the moon landing?', MY_CONFIG.LLM_MODEL)\n",
+    "res = query_engine.query(query)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "allycat-1",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

4_query.py

@@ -0,0 +1,194 @@
+"""
+Vector RAG Query
+"""
+
+import os
+from my_config import MY_CONFIG
+
+# If connection to https://huggingface.co/ failed, uncomment the following path
+os.environ['HF_ENDPOINT'] = MY_CONFIG.HF_ENDPOINT
+
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.core import Settings
+from llama_index.core import VectorStoreIndex, StorageContext
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from dotenv import load_dotenv
+from llama_index.llms.litellm import LiteLLM
+import query_utils
+import time
+import logging
+import json
+
+# Create logs directory if it doesn't exist
+os.makedirs('logs/query', exist_ok=True)
+
+logging.basicConfig(
+    level=logging.INFO, 
+    format='%(asctime)s - %(levelname)s - %(message)s', 
+    handlers=[
+        logging.FileHandler('logs/query/query_log.txt', mode='a'),  # Save to file
+        logging.StreamHandler()  # Also show in console
+    ],
+    force=True
+)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def run_query(query: str):
+    global query_engine
+    logger.info("-----------------------------------")
+    start_time = time.time()
+    query = query_utils.tweak_query(query, MY_CONFIG.LLM_MODEL)
+    logger.info (f"\nProcessing Query:\n{query}")
+    
+    # Get initial vector response
+    vector_response = query_engine.query(query)
+    vector_text = str(vector_response).strip()
+    
+    # Structured prompt 
+    structured_prompt = f"""Please provide a comprehensive, well-structured answer using the provided document information.
+
+Question: {query}
+
+Document Information:
+{vector_text}
+
+Instructions:
+1. Provide accurate, factual information based on the documents
+2. Structure your response clearly with proper formatting
+3. Be comprehensive yet concise
+4. Highlight key relationships and important details when relevant
+5. Use bullet points or sections when appropriate for clarity
+
+Please provide your answer:"""
+    
+    # Use structured prompt for final synthesis
+    res = query_engine.query(structured_prompt)
+    
+    end_time = time.time()
+    total_time = end_time - start_time
+    logger.info ( "-------"
+                 + f"\nResponse:\n{res}" 
+                 + f"\n\n⏱️ Total time: {total_time:.1f} seconds"
+                 + f"\n\nResponse Metadata:\n{json.dumps(res.metadata, indent=2)}" 
+                 + f"\nSource Nodes: {[node.node_id for node in res.source_nodes]}"
+                 )
+    logger.info("-----------------------------------")
+    
+    # Save response and metadata to files
+    _save_query_files(query, res, total_time)
+    
+    return res
+
+def _save_query_files(query: str, response, total_time: float):
+    """Save query response and metadata to files."""
+    import time
+    timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
+    
+    try:
+        # Save response to file
+        with open('logs/query/query_responses.txt', 'a', encoding='utf-8') as f:
+            f.write(f"\n{'='*80}\n")
+            f.write(f"QUERY [{timestamp}]: {query}\n")
+            f.write(f"{'='*80}\n")
+            f.write(f"RESPONSE: {response}\n")
+            f.write(f"TIME: {total_time:.1f} seconds\n")
+            f.write(f"{'='*80}\n\n")
+        
+        # Save metadata to file
+        with open('logs/query/query_metadata.txt', 'a', encoding='utf-8') as f:
+            f.write(f"\n{'='*80}\n")
+            f.write(f"METADATA [{timestamp}]: {query}\n")
+            f.write(f"{'='*80}\n")
+            f.write(f"TIME: {total_time:.1f} seconds\n")
+            f.write(json.dumps(response.metadata, indent=2, default=str))
+            f.write(f"\n{'='*80}\n\n")
+            
+        logger.info(f"Saved response and metadata for query: {query[:50]}...")
+    except Exception as e:
+        logger.error(f"Failed to save query files: {e}")
+
+## ======= end : run_query =======
+
+## load env config
+load_dotenv()
+
+# Setup embeddings
+Settings.embed_model = HuggingFaceEmbedding(
+    model_name = MY_CONFIG.EMBEDDING_MODEL
+)
+logger.info (f"✅ Using embedding model: {MY_CONFIG.EMBEDDING_MODEL}")
+
+# Connect to vector database based on configuration
+if MY_CONFIG.VECTOR_DB_TYPE == "cloud_zilliz":
+    # Use Zilliz Cloud
+    if not MY_CONFIG.ZILLIZ_CLUSTER_ENDPOINT or not MY_CONFIG.ZILLIZ_TOKEN:
+        raise ValueError("Cloud database configuration missing. Set ZILLIZ_CLUSTER_ENDPOINT and ZILLIZ_TOKEN in .env")
+    
+    vector_store = MilvusVectorStore(
+        uri=MY_CONFIG.ZILLIZ_CLUSTER_ENDPOINT,
+        token=MY_CONFIG.ZILLIZ_TOKEN,
+        dim=MY_CONFIG.EMBEDDING_LENGTH,
+        collection_name=MY_CONFIG.COLLECTION_NAME,
+        overwrite=False
+    )
+    storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    logger.info("Connected to cloud vector database")
+else:
+    # Use local Milvus (default)
+    vector_store = MilvusVectorStore(
+        uri=MY_CONFIG.MILVUS_URI_VECTOR,
+        dim=MY_CONFIG.EMBEDDING_LENGTH,
+        collection_name=MY_CONFIG.COLLECTION_NAME,
+        overwrite=False
+    )
+    storage_context = StorageContext.from_defaults(vector_store=vector_store)
+    logger.info("Connected to local vector database")
+
+# Load Document Index from database
+index = VectorStoreIndex.from_vector_store(
+    vector_store=vector_store, storage_context=storage_context)
+logger.info("Vector index loaded successfully")
+
+# Setup LLM
+logger.info (f"✅ Using LLM model : {MY_CONFIG.LLM_MODEL}")
+Settings.llm = LiteLLM (
+        model=MY_CONFIG.LLM_MODEL,
+    )
+ 
+query_engine = index.as_query_engine()
+
+# Sample queries
+queries = [
+    # "What is AI Alliance?",
+    # "What are the main focus areas of AI Alliance?",
+    # "What are some ai alliance projects?",
+    # "What are the upcoming events?", 
+    # "How do I join the AI Alliance?",
+    # "When was the moon landing?",
+]
+
+for query in queries:
+    run_query(query)
+
+logger.info("-----------------------------------")
+
+while True:
+    # Get user input
+    user_query = input("\nEnter your question (or 'q' to exit): ")
+    
+    # Check if user wants to quit
+    if user_query.lower() in ['quit', 'exit', 'q']:
+        logger.info ("Goodbye!")
+        break
+    
+    # Process the query
+    if user_query.strip() == "":
+        continue
+    
+    try:
+        run_query(user_query)
+    except Exception as e:
+        logger.error(f"Error processing query: {e}")
+        print(f"Error processing query: {e}")

4b_query_graph copy.py

@@ -0,0 +1,338 @@
+"""
+GraphRAG Implementation - Main Query Engine
+
+Imports Step 1 functionality from query-graph-functions/setup.py
+and implements the complete 25-step DRIFT search methodology.
+"""
+
+import time
+import logging
+import json
+import importlib
+import sys
+import os
+import asyncio
+from typing import Dict, Any
+
+# Apply nest_asyncio to allow nested event loops
+import nest_asyncio 
+nest_asyncio.apply()
+
+# Import Step 1 functionality from setup module
+from query_graph_functions.setup import create_graphrag_setup
+# Import Steps 3-5 functionality from query preprocessing module  
+from query_graph_functions.query_preprocessing import (
+    create_query_preprocessor,
+    preprocess_query_pipeline
+)
+# Import Steps 6-8 functionality from knowledge retrieval module
+from query_graph_functions.knowledge_retrieval import CommunitySearchEngine
+# Import Steps 9-12 functionality from follow-up search module
+from query_graph_functions.follow_up_search import FollowUpSearch
+# Import Steps 13-14 functionality from vector augmentation module
+from query_graph_functions.vector_augmentation import VectorAugmentationEngine
+# Import Steps 15-16 functionality from answer synthesis module
+from query_graph_functions.answer_synthesis import AnswerSynthesisEngine
+# Import Steps 17-20 functionality from response management module
+from query_graph_functions.response_management import ResponseManager
+from my_config import MY_CONFIG
+import query_utils
+
+# Configure logging - Save to file and console
+logging.basicConfig(
+    level=logging.INFO, 
+    format='%(asctime)s - %(levelname)s - %(message)s', 
+    handlers=[
+        logging.FileHandler('logs/graphrag_query/graphrag_query_log.txt', mode='a'),  # Save to file
+        logging.StreamHandler()  # Also show in console
+    ],
+    force=True
+)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Log session start
+logger.info("=" * 80)
+logger.info(f"GraphRAG Session Started - {time.strftime('%Y-%m-%d %H:%M:%S')}")
+logger.info("=" * 80)
+
+
+class GraphQueryEngine:
+    """
+    GraphRAG Query Engine - Complete Implementation
+    
+    Uses setup module for Step 1 initialization and query preprocessing 
+    module for Steps 3-5, implementing the full 25-step DRIFT search methodology.
+    """
+    
+    def __init__(self):
+        logger.info("GraphRAG Query Engine Initializing")
+        
+        # Initialize using setup module (Step 1)
+        self.setup = create_graphrag_setup()
+        
+        # Extract components from setup
+        self.neo4j_conn = self.setup.neo4j_conn
+        self.query_engine = self.setup.query_engine
+        self.graph_stats = self.setup.graph_stats
+        self.drift_config = self.setup.drift_config
+        self.llm = self.setup.llm
+        self.config = self.setup.config
+        
+        # Initialize query preprocessor (Steps 3-5) - will be created async
+        self.query_preprocessor = None
+        
+        # Initialize response manager (Steps 17-20)
+        self.response_manager = ResponseManager(self.setup)
+        
+        logger.info("GraphRAG Query Engine Ready")
+
+    async def run_query_async(self, user_query: str) -> Dict[str, Any]:
+        """
+        GraphRAG Query Pipeline - Main Entry Point (Async)
+        
+        Implements Phase B (Steps 3-5) of the 25-step DRIFT search methodology
+        """
+        logger.info("=" * 60)
+        logger.info("GraphRAG Query Pipeline Starting")
+        logger.info("=" * 60)
+        
+        start_time = time.time()
+        
+        # Apply query optimization
+        optimized_query = query_utils.tweak_query(user_query, MY_CONFIG.LLM_MODEL)
+        logger.info(f"Original Query: {user_query}")
+        if optimized_query != user_query:
+            logger.info(f"Optimized Query: {optimized_query}")
+        
+        try:
+            # Validate system readiness using setup module
+            if not self.setup.validate_system_readiness():
+                return self._generate_error_response("System not properly initialized")
+            
+            # PHASE B: QUERY PREPROCESSING (Steps 3-5)
+            logger.info("Phase B: Starting Query Preprocessing (Steps 3-5)")
+            
+            # Initialize query preprocessor if needed
+            if not self.query_preprocessor:
+                self.query_preprocessor = await create_query_preprocessor(
+                    self.config, self.graph_stats
+                )
+            
+            # Execute complete preprocessing pipeline
+            analysis, routing, vectorization = await preprocess_query_pipeline(
+                optimized_query, self.config, self.graph_stats
+            )
+            
+            logger.info(f"Phase B Completed: "
+                       f"Type={analysis.query_type.value}, "
+                       f"Strategy={routing.search_strategy.value}")
+            
+            # PHASE C: COMMUNITY RETRIEVAL (Steps 6-7)
+            logger.info("Phase C: Starting Community Retrieval (Steps 6-7)")
+            
+            # Create community search engine
+            community_engine = CommunitySearchEngine(self.setup)
+            
+            # Execute the primer phase (Steps 6-8)
+            community_results = await community_engine.execute_primer_phase(
+                vectorization.embedding, routing
+            )
+            
+            # Extract communities for Phase D
+            communities = community_results['communities']
+            
+            logger.info(f"Phase C Completed: Retrieved {len(communities)} communities")
+            
+            # PHASE D: FOLLOW-UP SEARCH (Steps 9-12)
+            logger.info("Phase D: Starting Follow-up Search (Steps 9-12)")
+            
+            # Create follow-up search engine
+            follow_up_engine = FollowUpSearch(self.setup)
+            
+            # Execute follow-up search phase
+            follow_up_results = await follow_up_engine.execute_follow_up_phase(
+                community_results, routing
+            )
+            
+            logger.info(f"Phase D Completed: Generated {len(follow_up_results.get('intermediate_answers', []))} detailed answers")
+            
+            # PHASE E: VECTOR SEARCH AUGMENTATION (Steps 13-14)
+            logger.info("Phase E: Starting Vector Search Augmentation (Steps 13-14)")
+            
+            # Create vector augmentation engine
+            vector_engine = VectorAugmentationEngine(self.setup)
+            
+            # Execute vector augmentation phase
+            augmentation_results = await vector_engine.execute_vector_augmentation_phase(
+                vectorization.embedding, 
+                {'communities': communities, 'initial_answer': community_results['initial_answer'], 'follow_up_results': follow_up_results},
+                routing
+            )
+            
+            logger.info(f"Phase E Completed: Vector augmentation confidence: {augmentation_results.augmentation_confidence:.3f}")
+            
+            # PHASE F: ANSWER SYNTHESIS (Steps 15-16)
+            logger.info("Phase F: Starting Answer Synthesis (Steps 15-16)")
+            
+            # Create answer synthesis engine
+            synthesis_engine = AnswerSynthesisEngine(self.setup)
+            
+            # Execute comprehensive answer synthesis
+            synthesis_results = await synthesis_engine.execute_answer_synthesis_phase(
+                analysis, routing, community_results, follow_up_results, augmentation_results
+            )
+            
+            logger.info(f"Phase F Completed: Final synthesis confidence: {synthesis_results.confidence_score:.3f}")
+            
+            # PHASE G: RESPONSE MANAGEMENT (Steps 17-20)
+            logger.info("Phase G: Starting Response Management (Steps 17-20)")
+            
+            # Enhanced implementation using preprocessing results
+            if self.query_engine:
+                # Use the vectorized query for better results  
+                _ = self.query_engine.query(vectorization.normalized_query)
+                total_time = time.time() - start_time
+                
+                logger.info(f"Enhanced Query Completed in {total_time:.2f}s")
+                logger.info("=" * 60)
+                
+                # Use Phase F synthesis result as the final answer
+                enhanced_answer = synthesis_results.final_answer
+                
+                # Generate comprehensive metadata using ResponseManager
+                metadata = self.response_manager.generate_metadata(
+                    analysis=analysis,
+                    routing=routing, 
+                    vectorization=vectorization,
+                    community_results=community_results,
+                    follow_up_results=follow_up_results,
+                    augmentation_results=augmentation_results,
+                    synthesis_results=synthesis_results,
+                    total_time=total_time,
+                    graph_stats=self.graph_stats,
+                    config=self.config
+                )
+                
+                result = {
+                    "answer": enhanced_answer,
+                    "metadata": metadata
+                }
+                
+                # Save response and metadata to files using ResponseManager
+                self.response_manager.save_response_to_files(user_query, result)
+                
+                logger.info("Phase G Completed: Response management finished")
+                
+                return result
+            else:
+                return await synthesis_engine.generate_error_response("Query engine not available")
+                
+        except Exception as e:
+            logger.error(f"Query Pipeline Failed: {e}")
+            return await synthesis_engine.generate_error_response(f"Query processing error: {e}")
+    
+    def run_query(self, user_query: str) -> Dict[str, Any]:
+        """
+        Synchronous wrapper for async query processing.
+        
+        This maintains backward compatibility while using the new async pipeline.
+        Uses nest_asyncio and our LiteLLM patch to properly handle async tasks.
+        """
+        try:
+            # Use the current event loop since nest_asyncio.apply() has been called
+            loop = asyncio.get_event_loop()
+            
+            # Create a future to gather all tasks and wait for completion
+            async def run_with_cleanup():
+                try:
+                    # Run the main query
+                    result = await self.run_query_async(user_query)
+                    
+                    # Use setup module's cleanup function
+                    await self.setup.cleanup_async_tasks(timeout=2.0)
+                    
+                    return result
+                except Exception as e:
+                    logger.error(f"Async Query Execution Failed: {e}")
+                    raise e
+                
+            # Run the async function with cleanup
+            return loop.run_until_complete(run_with_cleanup())
+            
+        except Exception as e:
+            logger.error(f"Sync Query Wrapper Failed: {e}")
+            # Use synthesis engine for error handling
+            synthesis_engine = AnswerSynthesisEngine(self.setup)
+            loop = asyncio.get_event_loop()
+            return loop.run_until_complete(
+                synthesis_engine.generate_error_response(f"Query processing error: {e}")
+            )
+
+    def close(self):
+        """Clean up connections using setup module"""
+        if self.setup:
+            self.setup.close()
+        logger.info("GraphQueryEngine cleanup complete")
+
+
+if __name__ == "__main__":
+    print("GraphRAG Implementation - Hot Reload Enabled")
+    print("=" * 50)
+    print("Step 1: Initialization and Connection")
+    print("Hot Reload: Type 'r' to reload modules")
+    print("=" * 50)
+    
+    engine = GraphQueryEngine()
+    
+    try:
+        # Create an event loop for the main thread
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        
+        while True:
+            user_query = input("\nEnter your question ('q' to exit, 'r' to reload): ")
+            
+            if user_query.lower() in ['quit', 'exit', 'q']:
+                print("Goodbye!")
+                break
+            
+            if user_query.lower() == 'r':
+                print("Reloading...")
+                engine.close()
+                
+                # Run cleanup tasks before reloading using setup module
+                loop.run_until_complete(
+                    engine.setup.cleanup_async_tasks(timeout=3.0) if engine.setup else None
+                )
+                
+                engine = GraphQueryEngine()
+                print("Reloaded!")
+                continue
+            
+            if user_query.strip() == "":
+                continue
+            
+            # Direct method call - clean forward-only implementation
+            result = engine.run_query(user_query)
+            
+            # Print results
+            print("\n" + "=" * 60)
+            print("GraphRAG Query Results")
+            print("=" * 60)
+            print(f"Answer: {result['answer']}")
+            print(f"\nMetadata: {json.dumps(result['metadata'], indent=2)}")
+            print("=" * 60)
+            
+    except Exception as e:
+        logger.error(f"Error processing query: {e}")
+        print(f"Error processing query: {e}")
+    finally:
+        # Run final cleanup before exiting using setup module
+        if 'loop' in locals() and 'engine' in locals():
+            loop.run_until_complete(
+                engine.setup.cleanup_async_tasks(timeout=5.0) if engine.setup else None
+            )
+            loop.close()
+        if 'engine' in locals():
+            engine.close()

4b_query_graph.py

@@ -0,0 +1,327 @@
+"""
+GraphRAG Query 
+"""
+
+import time
+import logging
+import json
+import importlib
+import sys
+import os
+import asyncio
+from typing import Dict, Any
+
+# Apply nest_asyncio to allow nested event loops
+import nest_asyncio 
+nest_asyncio.apply()
+
+# Import Step 1 functionality from setup module
+from query_graph_functions.setup import create_graphrag_setup
+# Import Steps 3-5 functionality from query preprocessing module  
+from query_graph_functions.query_preprocessing import (
+    create_query_preprocessor,
+    preprocess_query_pipeline
+)
+# Import Steps 6-8 functionality from knowledge retrieval module
+from query_graph_functions.knowledge_retrieval import CommunitySearchEngine
+# Import Steps 9-12 functionality from follow-up search module
+from query_graph_functions.follow_up_search import FollowUpSearch
+# Import Steps 13-14 functionality from vector augmentation module
+from query_graph_functions.vector_augmentation import VectorAugmentationEngine
+# Import Steps 15-16 functionality from answer synthesis module
+from query_graph_functions.answer_synthesis import AnswerSynthesisEngine
+# Import Steps 17-20 functionality from response management module
+from query_graph_functions.response_management import ResponseManager
+from my_config import MY_CONFIG
+import query_utils
+
+# Create logs directory if it doesn't exist
+os.makedirs('logs/graphrag_query', exist_ok=True)
+
+# Configure logging - Save to file and console
+logging.basicConfig(
+    level=logging.INFO, 
+    format='%(asctime)s - %(levelname)s - %(message)s', 
+    handlers=[
+        logging.FileHandler('logs/graphrag_query/graphrag_query_log.txt', mode='a'),  # Save to file
+        logging.StreamHandler()  # Also show in console
+    ],
+    force=True
+)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Log session start
+logger.info("=" * 80)
+logger.info(f"GraphRAG Session Started - {time.strftime('%Y-%m-%d %H:%M:%S')}")
+logger.info("=" * 80)
+
+
+class GraphQueryEngine:
+    def __init__(self):
+        logger.info("GraphRAG Query Engine Initializing")
+        
+        # Initialize using setup module (Step 1)
+        self.setup = create_graphrag_setup()
+        
+        # Extract components from setup
+        self.neo4j_conn = self.setup.neo4j_conn
+        self.query_engine = self.setup.query_engine
+        self.graph_stats = self.setup.graph_stats
+        self.drift_config = self.setup.drift_config
+        self.llm = self.setup.llm
+        self.config = self.setup.config
+        
+        # Initialize query preprocessor (Steps 3-5) - will be created async
+        self.query_preprocessor = None
+        
+        # Initialize response manager (Steps 17-20)
+        self.response_manager = ResponseManager(self.setup)
+        
+        logger.info("GraphRAG Query Engine Ready")
+
+    async def run_query_async(self, user_query: str) -> Dict[str, Any]:
+        """
+        GraphRAG Query Pipeline - Main Entry Point (Async)
+        
+        Implements Phase B (Steps 3-5) of the 25-step DRIFT search methodology
+        """
+        logger.info("=" * 60)
+        logger.info("GraphRAG Query Pipeline Starting")
+        logger.info("=" * 60)
+        
+        start_time = time.time()
+        
+        # Apply query optimization
+        optimized_query = query_utils.tweak_query(user_query, MY_CONFIG.LLM_MODEL)
+        logger.info(f"Original Query: {user_query}")
+        if optimized_query != user_query:
+            logger.info(f"Optimized Query: {optimized_query}")
+        
+        try:
+            # Validate system readiness using setup module
+            if not self.setup.validate_system_readiness():
+                return self._generate_error_response("System not properly initialized")
+            
+            # PHASE B: QUERY PREPROCESSING (Steps 3-5)
+            logger.info("Phase B: Starting Query Preprocessing (Steps 3-5)")
+            
+            # Initialize query preprocessor if needed
+            if not self.query_preprocessor:
+                self.query_preprocessor = await create_query_preprocessor(
+                    self.config, self.graph_stats
+                )
+            
+            # Execute complete preprocessing pipeline
+            analysis, routing, vectorization = await preprocess_query_pipeline(
+                optimized_query, self.config, self.graph_stats
+            )
+            
+            logger.info(f"Phase B Completed: "
+                       f"Type={analysis.query_type.value}, "
+                       f"Strategy={routing.search_strategy.value}")
+            
+            # PHASE C: COMMUNITY RETRIEVAL (Steps 6-7)
+            logger.info("Phase C: Starting Community Retrieval (Steps 6-7)")
+            
+            # Create community search engine
+            community_engine = CommunitySearchEngine(self.setup)
+            
+            # Execute the primer phase (Steps 6-8)
+            community_results = await community_engine.execute_primer_phase(
+                vectorization.embedding, routing
+            )
+            
+            # Extract communities for Phase D
+            communities = community_results['communities']
+            
+            logger.info(f"Phase C Completed: Retrieved {len(communities)} communities")
+            
+            # PHASE D: FOLLOW-UP SEARCH (Steps 9-12)
+            logger.info("Phase D: Starting Follow-up Search (Steps 9-12)")
+            
+            # Create follow-up search engine
+            follow_up_engine = FollowUpSearch(self.setup)
+            
+            # Execute follow-up search phase
+            follow_up_results = await follow_up_engine.execute_follow_up_phase(
+                community_results, routing
+            )
+            
+            logger.info(f"Phase D Completed: Generated {len(follow_up_results.get('intermediate_answers', []))} detailed answers")
+            
+            # PHASE E: VECTOR SEARCH AUGMENTATION (Steps 13-14)
+            logger.info("Phase E: Starting Vector Search Augmentation (Steps 13-14)")
+            
+            # Create vector augmentation engine
+            vector_engine = VectorAugmentationEngine(self.setup)
+            
+            # Execute vector augmentation phase
+            augmentation_results = await vector_engine.execute_vector_augmentation_phase(
+                vectorization.embedding, 
+                {'communities': communities, 'initial_answer': community_results['initial_answer'], 'follow_up_results': follow_up_results},
+                routing
+            )
+            
+            logger.info(f"Phase E Completed: Vector augmentation confidence: {augmentation_results.augmentation_confidence:.3f}")
+            
+            # PHASE F: ANSWER SYNTHESIS (Steps 15-16)
+            logger.info("Phase F: Starting Answer Synthesis (Steps 15-16)")
+            
+            # Create answer synthesis engine
+            synthesis_engine = AnswerSynthesisEngine(self.setup)
+            
+            # Execute comprehensive answer synthesis
+            synthesis_results = await synthesis_engine.execute_answer_synthesis_phase(
+                analysis, routing, community_results, follow_up_results, augmentation_results
+            )
+            
+            logger.info(f"Phase F Completed: Final synthesis confidence: {synthesis_results.confidence_score:.3f}")
+            
+            # PHASE G: RESPONSE MANAGEMENT (Steps 17-20)
+            logger.info("Phase G: Starting Response Management (Steps 17-20)")
+            
+            # Enhanced implementation using preprocessing results
+            if self.query_engine:
+                # Use the vectorized query for better results  
+                _ = self.query_engine.query(vectorization.normalized_query)
+                total_time = time.time() - start_time
+                
+                logger.info(f"Enhanced Query Completed in {total_time:.2f}s")
+                logger.info("=" * 60)
+                
+                # Use Phase F synthesis result as the final answer
+                enhanced_answer = synthesis_results.final_answer
+                
+                # Generate comprehensive metadata using ResponseManager
+                metadata = self.response_manager.generate_comprehensive_metadata(
+                    analysis=analysis,
+                    routing=routing, 
+                    vectorization=vectorization,
+                    community_results=community_results,
+                    follow_up_results=follow_up_results,
+                    augmentation_results=augmentation_results,
+                    synthesis_results=synthesis_results,
+                    total_time=total_time
+                )
+                
+                result = {
+                    "answer": enhanced_answer,
+                    "metadata": metadata
+                }
+                
+                # Save response and metadata to files using ResponseManager
+                self.response_manager.save_response_to_files(user_query, result)
+                
+                logger.info("Phase G Completed: Response management finished")
+                
+                return result
+            else:
+                return synthesis_engine.generate_error_response("Query engine not available")
+                
+        except Exception as e:
+            logger.error(f"Query Pipeline Failed: {e}")
+            synthesis_engine = AnswerSynthesisEngine(self.setup)
+            return synthesis_engine.generate_error_response(f"Query processing error: {e}")
+    
+    def run_query(self, user_query: str) -> Dict[str, Any]:
+        """
+        Synchronous wrapper for async query processing.
+        
+        This maintains backward compatibility while using the new async pipeline.
+        Uses nest_asyncio and our LiteLLM patch to properly handle async tasks.
+        """
+        try:
+            # Use the current event loop since nest_asyncio.apply() has been called
+            loop = asyncio.get_event_loop()
+            
+            # Create a future to gather all tasks and wait for completion
+            async def run_with_cleanup():
+                try:
+                    # Run the main query
+                    result = await self.run_query_async(user_query)
+                    
+                    # Use setup module's cleanup function
+                    await self.setup.cleanup_async_tasks(timeout=2.0)
+                    
+                    return result
+                except Exception as e:
+                    logger.error(f"Async Query Execution Failed: {e}")
+                    raise e
+                
+            # Run the async function with cleanup
+            return loop.run_until_complete(run_with_cleanup())
+            
+        except Exception as e:
+            logger.error(f"Sync Query Wrapper Failed: {e}")
+            # Use synthesis engine for error handling
+            synthesis_engine = AnswerSynthesisEngine(self.setup)
+            return synthesis_engine.generate_error_response(f"Query processing error: {e}")
+
+    def close(self):
+        """Clean up connections using setup module"""
+        if self.setup:
+            self.setup.close()
+        logger.info("GraphQueryEngine cleanup complete")
+
+
+if __name__ == "__main__":
+    print("GraphRAG Implementation - Hot Reload Enabled")
+    print("=" * 50)
+    print("Step 1: Initialization and Connection")
+    print("Hot Reload: Type 'r' to reload modules")
+    print("=" * 50)
+    
+    engine = GraphQueryEngine()
+    
+    try:
+        # Create an event loop for the main thread
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        
+        while True:
+            user_query = input("\nEnter your question ('q' to exit, 'r' to reload): ")
+            
+            if user_query.lower() in ['quit', 'exit', 'q']:
+                print("Goodbye!")
+                break
+            
+            if user_query.lower() == 'r':
+                print("Reloading...")
+                engine.close()
+                
+                # Run cleanup tasks before reloading using setup module
+                loop.run_until_complete(
+                    engine.setup.cleanup_async_tasks(timeout=3.0) if engine.setup else None
+                )
+                
+                engine = GraphQueryEngine()
+                print("Reloaded!")
+                continue
+            
+            if user_query.strip() == "":
+                continue
+            
+            # Direct method call - clean forward-only implementation
+            result = engine.run_query(user_query)
+            
+            # Print results
+            print("\n" + "=" * 60)
+            print("GraphRAG Query Results")
+            print("=" * 60)
+            print(f"Answer: {result['answer']}")
+            print(f"\nMetadata: {json.dumps(result['metadata'], indent=2)}")
+            print("=" * 60)
+            
+    except Exception as e:
+        logger.error(f"Error processing query: {e}")
+        print(f"Error processing query: {e}")
+    finally:
+        # Run final cleanup before exiting using setup module
+        if 'loop' in locals() and 'engine' in locals():
+            loop.run_until_complete(
+                engine.setup.cleanup_async_tasks(timeout=5.0) if engine.setup else None
+            )
+            loop.close()
+        if 'engine' in locals():
+            engine.close()

CHANGELOG.md

@@ -0,0 +1,208 @@
+# AllyCat Changelog
+
+All notable technical changes to AllyCat GraphRAG will be documented in this file.
+
+## [Unreleased]
+
+### Added
+
+#### GraphRAG Implementation
+- **GraphRAG Core System**: Implemented Microsoft GraphRAG-inspired architecture
+  - Entity extraction and relationship mapping from documents
+  - Community detection algorithms for knowledge graph clustering
+  - Multi-phase graph processing pipeline (phase 1: entities/relationships, phase 2: communities, phase 3: summaries)
+  - Graph-based query system with hierarchical summarization
+  - Neo4j integration for graph database storage (`3b_save_to_graph_db.py`)
+  - Graph query functions in `query_graph_functions/` directory
+  - Dual RAG modes: Traditional Vector RAG + Advanced GraphRAG
+  - **Note**: More improvements planned based on [Microsoft GraphRAG Project](https://github.com/microsoft/graphrag)
+
+#### LLM Provider Support
+- **Cerebras API Integration**: Added support for Cerebras ultra-fast inference
+- **Google Gemini API Integration**: Added support for Google's Gemini models
+- **LiteLLM Framework**: Implemented `litellm_patch.py` for unified LLM API interface
+  - Supports multiple providers: OpenAI, Replicate, Nebius, Cerebras, Gemini, Anthropic, and more
+  - Simplified provider switching via environment variables
+
+#### Database Solutions
+- **Zilliz Cloud Integration**: Added cloud-based vector database support
+  - Implemented `3_save_to_vector_db_zilliz.py` for Zilliz Cloud
+  - Cloud vector database eliminates need for local Milvus server
+  - Configurable via `VECTOR_DB_TYPE` environment variable
+- **Neo4j Graph Database**: Integrated for GraphRAG knowledge graph storage
+  - Stores entities, relationships, and community structures
+  - Enables complex graph traversal queries
+
+#### Docker Deployment System
+- **Three Deployment Modes**: Flexible deployment configurations
+  - **Cloud Mode** (`docker-compose.cloud.yml`): Cloud LLM + Cloud Zilliz vector DB
+  - **Hybrid Mode** (`docker-compose.hybrid.yml`): Cloud LLM + Local Milvus
+  - **Local Mode** (`docker-compose.local.yml`): Local Ollama + Local Milvus
+- **Automated Deployment Script**: `docker-startup.sh` orchestrates full deployment
+  - Conditional service startup based on deployment mode
+  - Automatic Ollama model download for local mode
+  - Smart service detection and initialization
+
+#### Automatic Pipeline Execution
+- **End-to-End Automation**: Single command deployment from crawling to running application
+  - Automatic website crawling when `WEBSITE_URL` is set
+  - Sequential pipeline execution: crawl → process → vector DB → graph processing → graph DB
+  - Automatic application startup after pipeline completion
+  - Controlled via `AUTO_RUN_PIPELINE` environment variable
+  - **User Action Required**: Only set environment variables in `.env` file
+
+#### Document Processing Improvements
+- **HTML/HTM Processing**: Switched to `html2text` library for better HTML parsing
+  - Improved markdown conversion quality
+  - Better handling of HTML structure and formatting
+  - **Resolves**: [Issue #50](https://github.com/The-AI-Alliance/allycat/issues/50)
+- **PDF Processing**: Integrated `docling` library for advanced PDF parsing
+  - High-quality PDF to markdown conversion
+  - Preserves document structure and formatting
+  - Handles complex PDF layouts
+
+#### Port Management System
+- **Multiple Dynamic Ports**: Flexible port configuration for different services
+  - `FLASK_GRAPH_PORT=8080` - Flask GraphRAG application
+  - `FLASK_VECTOR_PORT=8081` - Flask Vector RAG application
+  - `CHAINLIT_GRAPH_PORT=8083` - Chainlit GraphRAG application
+  - `CHAINLIT_VECTOR_PORT=8082` - Chainlit Vector RAG application
+  - `DOCKER_PORT` - Host machine port mapping
+  - `DOCKER_APP_PORT=8080` - Internal container port
+  - `OLLAMA_PORT=11434` - Ollama server port (local mode)
+- **Smart Port Routing**: Automatic port selection based on `APP_TYPE` environment variable
+  - Supports: `flask_graph`, `flask`, `chainlit_graph`, `chainlit`
+
+#### Memory Optimization System
+- **CLEANUP_PIPELINE_DEPS Feature**: Post-pipeline dependency cleanup
+  - Created `requirements-runtime.txt` (~300 MB) - minimal packages for running Flask GraphRAG app
+  - Created `requirements-build.txt` (~500 MB) - pipeline-only packages that can be removed
+  - Created `cleanup_pipeline_deps.sh` - automated cleanup script
+  - Integrated cleanup into `docker-startup.sh` with conditional execution
+  - Added `CLEANUP_PIPELINE_DEPS` configuration to all `.env` sample files
+  - Created comprehensive technical documentation in `docs/docker-memory-optimization.md`
+  - Updated `docs/docker-deployment-guide.md` with memory optimization section
+  - **Benefits**: Reduces container RAM from ~800 MB to ~300 MB, enabling 1GB deployments
+  - **Cost Savings**: DigitalOcean 1GB ($12/mo) vs 2GB ($25/mo) = $156/year savings (52% reduction)
+
+### Changed
+- **Chainlit Port Configuration**: Reverted Chainlit apps to use default port behavior
+  - Removed custom port configuration code from Python files
+  - Chainlit now uses default port 8000 for native Python execution
+  - Docker deployments use custom ports via `--port` flag in `docker-startup.sh`
+  - Updated documentation in `docs/graphrag-demo/Setup.md`, `docs/configuration.md`, and `my_config.py`
+  - Native Python: `chainlit run app_chainlit_graph.py` (port 8000) or with custom `--port 8083`
+  - Docker: Custom ports 8082 (vector) and 8083 (graph) configured via environment variables
+
+
+### Fixed
+- **HTML/HTM File Processing**: Fixed HTML parsing issues ([Issue #50](https://github.com/The-AI-Alliance/allycat/issues/50))
+  - Switched from previous parser to `html2text` library
+  - Improved markdown conversion quality and reliability
+- **Pipeline Error Handling**: Confirmed robust error handling with `|| echo "Warning..."` pattern
+  - Cleanup script runs even if pipeline steps fail
+  - Application starts successfully regardless of pipeline completion status
+
+## [Previous Versions]
+
+### 2025-07-14: Major Update
+
+#### Added
+- **Robust Web Crawler** ([#31](https://github.com/The-AI-Alliance/allycat/issues/31))
+  - Complete rewrite of web crawler
+  - More robust handling of edge cases
+  - Support for multiple file types (not just text/html)
+  - Correct handling of anchor tags (`a.html#news`) in HTML files
+  - Customizable pause between requests to avoid hammering webservers
+  - Fixed issue with repeatedly downloading same content
+
+- **LiteLLM Integration** ([#34](https://github.com/The-AI-Alliance/allycat/issues/34))
+  - Unified LLM backend support replacing Replicate and Ollama setup
+  - Seamless access to local LLMs (using Ollama) and cloud inference providers
+  - Support for providers: Nebius, Replicate, and more
+  - Significantly simplified LLM configuration
+  - Added `python-dotenv` for environment variable management
+
+- **Expanded File Type Support** ([#37](https://github.com/The-AI-Alliance/allycat/issues/37))
+  - Support for PDF, DOCX, and other popular file types (previously only HTML)
+  - Integration with [Docling](https://github.com/docling-project/docling) for file processing
+  - Fixed issue with PDF downloads ([#35](https://github.com/The-AI-Alliance/allycat/issues/35))
+  - Processing all downloaded file types
+  - Updated process_file script
+
+- **UV Package Manager Support** ([#26](https://github.com/The-AI-Alliance/allycat/issues/26))
+  - Added [uv](https://docs.astral.sh/uv/) project structure
+  - Updated documentation for uv
+  - Continued support for `requirements.txt` and other package managers
+
+- **Better Config Management** ([#19](https://github.com/The-AI-Alliance/allycat/issues/19))
+  - User configuration via `.env` file
+  - Simplified config management
+  - Easier experimentation without code changes
+  - Documented configuration options
+  - Updated env.sample file with settings
+
+- **Metrics Collection**: Added metrics collection scripts and issue templates
+
+#### Changed
+- **Chainlit App Updates** ([#38](https://github.com/The-AI-Alliance/allycat/issues/38))
+  - Updated Chainlit application
+  - Customizable starter prompts
+- **Logo Updates** ([#39](https://github.com/The-AI-Alliance/allycat/issues/39))
+  - Updated logo to AllyCAT
+- **App Naming**: Changed Flask and Chainlit app names for clarity
+- Code cleanup improvements
+- Documentation updates across the project
+
+### 2025-05: Chainlit Integration
+
+#### Added
+- **Chainlit Chat Interface** ([#17](https://github.com/The-AI-Alliance/allycat/issues/17))
+  - Introduced Chainlit-based chat interface as alternative to Flask UI
+  - Improved chat UI experience
+
+#### Changed
+- Updated README with license and issues links
+
+### 2025-04: Dockerization and Local LLM Support
+
+#### Added
+- **Docker Support**: Complete dockerization of the application
+  - Docker deployment configurations
+  - Updated Google Cloud deployment guide
+  - Comprehensive Docker documentation (`running-in-docker.md`)
+- **Ollama Integration**: Local LLM support with Ollama
+  - Local LLM configuration and setup
+  - Local Jupyter Lab support
+  - Small tweaks to local LLM config
+- **Python Scripts**: Added Python script versions of notebooks
+
+#### Changed
+- Updated deployment documentation
+- Native running documentation (`running-natively.md`)
+
+### 2025-03: Database and LLM Updates
+
+#### Added
+- **Weaviate Database**: Added Weaviate vector database support
+- **Local LLM Support**: Initial local LLM integration
+
+#### Changed
+- **LLM Switch**: Changed from initial LLM to Llama
+- Added logo and GitHub link to UI
+- README and deploy guide updates
+
+### 2025-02: Initial Release - AllyCAT (formerly AllyChat)
+
+#### Added
+- **Initial Vector RAG System**: First version of AllyCAT
+  - Basic RAG implementation
+  - Vector database for document storage and retrieval
+  - Query system for document Q&A
+- **Flask Web Interface**: Web-based chat interface
+- **Basic Crawling**: Initial website crawling functionality
+- **Document Processing**: Basic document processing pipeline
+
+
+
+---

Dockerfile

@@ -0,0 +1,66 @@
+FROM python:3.11-slim
+
+# Build arguments for conditional installation
+ARG INSTALL_OLLAMA=false
+ARG INSTALL_LOCAL_VECTOR_DB=false
+
+# Set working directory
+WORKDIR /allycat
+
+# Set environment variables - Cloud-first defaults
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    LLM_RUN_ENV=cloud \
+    VECTOR_DB_TYPE=cloud_zilliz
+
+
+# Install minimal system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash \
+    curl \
+    git \
+    netcat-traditional \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements file - Use cloud-optimized by default
+COPY requirements-docker-cloud.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements-docker-cloud.txt
+
+# Conditional: Install Ollama only if requested
+RUN if [ "$INSTALL_OLLAMA" = "true" ]; then \
+        echo "Installing Ollama for local LLM support..."; \
+        curl -fsSL https://ollama.com/install.sh | sh; \
+    else \
+        echo "Skipping Ollama installation - using cloud LLM mode"; \
+    fi
+
+# Conditional: Install local vector DB dependencies
+RUN if [ "$INSTALL_LOCAL_VECTOR_DB" = "true" ]; then \
+        echo "Installing milvus-lite for local vector database..."; \
+        pip install --no-cache-dir milvus-lite==2.4.11; \
+    else \
+        echo "Skipping local vector DB - using Zilliz Cloud"; \
+    fi
+
+# Copy project files
+COPY . .
+RUN chmod +x ./docker-startup.sh
+
+# Cleanup unnecessary files
+RUN rm -rf .env workspace/* __pycache__ *.pyc
+
+# Expose all application ports (EXPOSE doesn't support env variables at build time)
+# Port 8080 = FLASK_GRAPH_PORT (default) / DOCKER_APP_PORT (default)
+# Port 8081 = FLASK_VECTOR_PORT (default)
+# Port 8082 = CHAINLIT_VECTOR_PORT (default)
+# Port 8083 = CHAINLIT_GRAPH_PORT (default)
+# Port mapping controlled by docker-compose.yml: ${DOCKER_PORT}:${DOCKER_APP_PORT}
+EXPOSE 8080 8081 8082 8083
+# Port 11434 = OLLAMA_PORT (default) - only used if INSTALL_OLLAMA=true
+EXPOSE 11434
+
+ENTRYPOINT ["./docker-startup.sh"]
+CMD ["deploy"]

Dockerfile-dev

@@ -0,0 +1,65 @@
+FROM python:3.11-slim
+
+# Build arguments for conditional installation
+ARG INSTALL_OLLAMA=false
+ARG INSTALL_LOCAL_VECTOR_DB=false
+
+# Set working directory
+WORKDIR /allycat
+
+# Set environment variables - Cloud-first defaults
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    LLM_RUN_ENV=cloud \
+    VECTOR_DB_TYPE=cloud_zilliz
+
+
+# Install minimal system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash \
+    curl \
+    git \
+    netcat-traditional \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements file - Use cloud-optimized by default
+COPY requirements-docker-cloud.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements-docker-cloud.txt
+
+# Conditional: Install Ollama only if requested
+RUN if [ "$INSTALL_OLLAMA" = "true" ]; then \
+        echo "Installing Ollama for local LLM support..."; \
+        curl -fsSL https://ollama.com/install.sh | sh; \
+    else \
+        echo "Skipping Ollama installation - using cloud LLM mode"; \
+    fi
+
+# Conditional: Install local vector DB dependencies
+RUN if [ "$INSTALL_LOCAL_VECTOR_DB" = "true" ]; then \
+        echo "Installing milvus-lite for local vector database..."; \
+        pip install --no-cache-dir milvus-lite==2.4.11; \
+    else \
+        echo "Skipping local vector DB - using Zilliz Cloud"; \
+    fi
+
+# Copy project files
+COPY . .
+RUN chmod +x ./docker-startup.sh
+
+# Cleanup unnecessary files
+RUN rm -rf .env workspace/* __pycache__ *.pyc
+
+# Expose all application ports (EXPOSE doesn't support env variables at build time)
+# Port 8080 = FLASK_GRAPH_PORT (default) / DOCKER_APP_PORT (default)
+# Port 8081 = FLASK_VECTOR_PORT (default)
+# Port 8082 = CHAINLIT_VECTOR_PORT (default)
+# Port 8083 = CHAINLIT_GRAPH_PORT (default)
+# Port mapping controlled by docker-compose.yml: ${DOCKER_PORT}:${DOCKER_APP_PORT}
+EXPOSE 8080 8081 8082 8083
+# Port 11434 = OLLAMA_PORT (default) - only used if INSTALL_OLLAMA=true
+EXPOSE 11434
+
+ENTRYPOINT ["./docker-startup.sh"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,10 +1,75 @@
----
-title: Allycat
-emoji: 🐨
-colorFrom: green
-colorTo: gray
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<img src="assets/allycat.png" alt="Alley Cat" width="200"/>
+
+[![License](https://img.shields.io/github/license/The-AI-Alliance/allycat)](https://github.com/The-AI-Alliance/allycat/blob/main/LICENSE)
+[![Issues](https://img.shields.io/github/issues/The-AI-Alliance/allycat)](https://github.com/The-AI-Alliance/allycat/issues)
+![GitHub stars](https://img.shields.io/github/stars/The-AI-Alliance/allycat?style=social)
+
+# AllyCat
+
+**AllyCat** is a full stack, open source chatbot that uses GenAI LLMs to answer questions about your website with both traditional Vector RAG and advanced GraphRAG capabilities. It is simple by design and will run on your laptop or server. 
+
+## Why?
+
+AllyCat is purposefully simple so it can be used by developers to learn how RAG-based GenAI works. Yet it is powerful enough to use with your website. You may also extend it for your own purposes. 
+
+⭐ **Found this tool helpful? Give it a star on GitHub to support the project and help others discover it!**
+
+**🗞️ [Allycat news](news.md)** - releases and new features!
+
+## How does it work? 
+AllyCat uses your choice of LLM and vector database to implement a chatbot written in Python using [RAG](https://en.wikipedia.org/wiki/Retrieval-augmented_generation) architecture.
+AllyCat also includes web scraping tools that extract data from your website (or any website). 
+
+## 🌟🌟 Features 🌟🌟 
+
+1. **Dual RAG Modes:** Traditional Vector RAG and Advanced GraphRAG with entity extraction and community detection
+2. **Web Crawling & Scraping:** Text extraction, data/HTML processing, conversion to markdown.
+   - **Currently uses:** [Data Prep Kit Connector](https://github.com/data-prep-kit/data-prep-kit/blob/dev/data-connector-lib/doc/overview.md) and [Docling](https://github.com/docling-project/docling)
+3. **Processing:** Chunking, vector embedding creation, saving to vector database.
+   - **Currently uses:** [Llama Index](https://docs.llamaindex.ai/en/stable/) and [Granite Embedding Model](https://huggingface.co/ibm-granite/granite-embedding-30m-english)
+4. **Multiple LLM Support:**
+   - **Local:** [Ollama](https://ollama.com/) with [Llama](https://www.llama.com) or [Granite](https://huggingface.co/collections/ibm-granite/granite-33-language-models-67f65d0cca24bcbd1d3a08e3)
+   - **Cloud:** OpenAI, Cerebras, Google Gemini, Replicate, Nebius, and more via [LiteLLM](https://docs.litellm.ai/docs)
+5. **Multiple Database Support:**
+   - **Vector:** [Milvus](https://milvus.io/) (local/embedded) or [Zilliz Cloud](https://zilliz.com/)
+   - **Graph:** [Neo4j](https://neo4j.com/) for GraphRAG knowledge graphs
+6. **Flexible Deployment:** Docker support with 3 modes (Cloud, Hybrid, Local) + Native Python
+7. **Chatbot Interfaces:** Flask web UI and Chainlit chat interface
+
+## ⚡️⚡️Quickstart ⚡️⚡️
+
+There are two ways to run Allycat.
+
+### Option 1: Use the Docker image
+
+A great option for a quick evaluation.  
+See [running AllyCat using docker](docs/running-in-docker.md)
+
+### Option 2: Run natively (for tweaking, developing)
+
+Choose this option if you want to tweak AllyCat to fit your needs. For example, experimenting with embedding models or LLMs.  
+See [running AllyCat natively](docs/running-natively.md)
+
+## AllyCat Workflow
+
+![](assets/rag-website-1.png)
+
+See [running allycat](docs/running-allycat.md)
+
+## Customizing AllyCat
+
+See [customizing allycat](docs/customizing-allycat.md)
+
+## Deploying AllyCat
+
+See [deployment guide](docs/deploy.md)
+
+## Developing AllyCat
+
+See [developing allycat](docs/developing-allycat.md)
+
+## Why the name **AllyCat**?
+
+Originally AllianceChat, we shortened it to AllyCat when we learned chat means cat in French. Who doesn't love cats?!
+
+

app_chainlit.py

@@ -0,0 +1,299 @@
+import chainlit as cl
+import os
+import logging
+from dotenv import load_dotenv
+import time
+import asyncio
+import re
+import logging
+
+# Import llama-index and related libraries
+from llama_index.core import VectorStoreIndex, StorageContext
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.core import Settings
+from llama_index.llms.litellm import LiteLLM
+from my_config import MY_CONFIG
+import query_utils
+
+# Global variables for LLM and index
+vector_index = None
+initialization_complete = False
+
+# Create logs directory if it doesn't exist
+os.makedirs('logs/chainlit', exist_ok=True)
+
+logging.basicConfig(level=logging.WARNING, 
+                    format='%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s',
+                    handlers=[
+                        logging.FileHandler('logs/chainlit/chainlit_vector.log', mode='a'),
+                        logging.StreamHandler()
+                    ],
+                    force=True)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+# Load environment variables from .env file
+load_dotenv()
+
+def initialize():
+    """
+    Initialize LLM and Milvus vector database using llama-index.
+    This function sets up the necessary components for the chat application.
+    """
+    global vector_index, initialization_complete
+    
+    if initialization_complete:
+        return
+    
+    logger.info("Initializing LLM and vector database...")
+    
+    # raise Exception ("init exception test") # debug
+    
+    try:
+        ## embedding model
+        Settings.embed_model = HuggingFaceEmbedding(
+            model_name = MY_CONFIG.EMBEDDING_MODEL
+        )
+        logger.info(f"✅ Using embedding model: {MY_CONFIG.EMBEDDING_MODEL}")
+
+        # Setup LLM
+        logger.info(f"✅ Using LLM model : {MY_CONFIG.LLM_MODEL}")
+        Settings.llm = LiteLLM(
+            model=MY_CONFIG.LLM_MODEL,
+            )
+        
+        # Initialize Milvus vector store
+        vector_store = MilvusVectorStore(
+            uri = MY_CONFIG.MILVUS_URI_VECTOR,
+            dim = MY_CONFIG.EMBEDDING_LENGTH , 
+            collection_name = MY_CONFIG.COLLECTION_NAME,
+            overwrite=False  # so we load the index from db
+        )
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        logger.info  (f"✅ Connected to Milvus instance: {MY_CONFIG.MILVUS_URI_VECTOR}")
+        
+        vector_index = VectorStoreIndex.from_vector_store(
+            vector_store=vector_store, storage_context=storage_context)
+        logger.info  (f"✅ Loaded index from vector db: {MY_CONFIG.MILVUS_URI_VECTOR}")
+
+        logger.info("Successfully initialized LLM and vector database")
+
+        initialization_complete = True
+    except Exception as e:
+        initialization_complete = False
+        logger.error(f"Error initializing LLM and vector database: {str(e)}")
+        raise (e)
+        # return False
+## -------------
+
+def extract_thinking_section(response_text):
+    """
+    Extract thinking section from LLM response if present.
+    
+    Args:
+        response_text (str): The full response from the LLM
+        
+    Returns:
+        tuple: (thinking_content, cleaned_response)
+            - thinking_content: Content within <think></think> tags or None if not found
+            - cleaned_response: Response with thinking section removed
+    """
+    thinking_pattern = r'<think>(.*?)</think>'
+    match = re.search(thinking_pattern, response_text, re.DOTALL)
+    
+    if match:
+        thinking_content = match.group(1).strip()
+        cleaned_response = re.sub(thinking_pattern, '', response_text, flags=re.DOTALL).strip()
+        return thinking_content, cleaned_response
+    else:
+        return None, response_text
+
+async def get_llm_response(message):
+    """
+    Process the user message and get a response from the LLM using Vector RAG
+    with structured prompt
+    """
+    global vector_index, initialization_complete
+    
+    # Check if LLM and index are initialized
+    if vector_index is None or  initialization_complete is None:
+        return "System did not initialize. Please try again later.", 0
+    
+    start_time = time.time()
+    response_text = ''
+    
+    try:
+        # Step 1: Query preprocessing
+        async with cl.Step(name="Query Preprocessing", type="tool") as step:
+            logger.info("Start query preprocessing step...")
+            step.input = message
+            
+            # Create a query engine from the index
+            query_engine = vector_index.as_query_engine()
+            
+            # Preprocess the query
+            original_message = message
+            message = query_utils.tweak_query(message, MY_CONFIG.LLM_MODEL)
+            
+            step.output = f"Optimized query: {message}"
+        ## --- end: Step 1 ---
+        
+        # Query the index with structured prompting
+        logger.info("Calling LLM with structured prompting...")
+        t1 = time.time()
+        
+        # Get initial vector response
+        vector_response = query_engine.query(message)
+        vector_text = str(vector_response).strip()
+        
+        # Structured prompt
+        structured_prompt = f"""Please provide a comprehensive, well-structured answer using the provided document information.
+
+Question: {message}
+
+Document Information:
+{vector_text}
+
+Instructions:
+1. Provide accurate, factual information based on the documents
+2. Structure your response clearly with proper formatting
+3. Be comprehensive yet concise
+4. Highlight key relationships and important details when relevant
+5. Use bullet points or sections when appropriate for clarity
+
+Please provide your answer:"""
+        
+        # Use structured prompt for final synthesis
+        response = query_engine.query(structured_prompt)
+        
+        t2 = time.time()
+        if response:
+            response_text = str(response).strip()
+        else:
+            response_text = "No response from LLM."
+        logger.info(f"LLM response received in {(t2 - t1):.2f} seconds:\n{response_text[:200]}")
+
+        # Step 2: Vector search and retrieval
+        async with cl.Step(name="Document Retrieval", type="retrieval") as step:
+            step.input = message
+
+            # Show retrieved documents
+            if hasattr(response, 'source_nodes') and response.source_nodes:
+                sources_output = []
+                for i, node in enumerate(response.source_nodes[:3]):  # Show top 3 sources
+                    score = node.score if hasattr(node, 'score') else 'N/A'
+                    text_preview = node.text[:200] + "..." if len(node.text) > 200 else node.text
+                    sources_output.append(f"Source {i+1} (Score: {score}): {text_preview}")
+                step.output = "\n\n".join(sources_output)
+            else:
+                step.output = "No relevant documents found."
+        
+        
+        # Extract thinking section if present
+        thinking_content, cleaned_response = extract_thinking_section(response_text)
+        # print (f"------ Thinking Content:-----\n{thinking_content}\n------")  # Debug print
+        # print (f"------ Cleaned Response:-----\n{cleaned_response}\n------")  # Debug print
+        
+        # Step 3: Optional Thinking Process
+        if thinking_content:
+            async with cl.Step(name="💭 Thinking Process", type="run") as step:
+                step.input = ""
+                step.output = thinking_content
+                logger.info(f"Thinking:\n{thinking_content[:200]}...")
+
+        # Step 4: LLM Answer
+        async with cl.Step(name="Response", type="llm") as step:
+            step.input = f"Query: {message}\nContext: Retrieved from vector database"
+            
+            if cleaned_response:
+                step.output = cleaned_response
+                logger.info(f"Response:\n{cleaned_response[:200]}...")
+            else:
+                step.output = "No response from LLM."
+                logger.info(f"Response:\nNo response from LLM.")
+
+    except Exception as e:
+        logger.error(f"Error getting LLM response: {str(e)}")
+        response_text =  f"Sorry, I encountered an error while processing your request:\n{str(e)}"
+        
+    end_time = time.time()
+    elapsed_time = end_time - start_time
+    
+    return response_text, elapsed_time
+    
+## --- end: def get_llm_response():
+
+# ====== CHAINLIT SPECIFIC CODE ======
+
+@cl.set_starters
+async def set_starters():
+    starters = []
+    for prompt in MY_CONFIG.STARTER_PROMPTS:
+        starters.append(
+            cl.Starter(
+                label=prompt.strip(),
+                message=prompt.strip(),
+            )
+        )
+    return starters
+## --- end: def set_starters(): ---
+
+@cl.on_chat_start
+async def start():
+    """Initialize the chat session"""
+    # Store initialization state in user session
+    cl.user_session.set("chat_started", True)
+    logger.info("User chat session started")
+    init_error = None
+    
+    try:
+        initialize()
+        # await cl.Message(content="How can I assist you today?").send()
+    except Exception as e:
+        init_error = str(e)
+        error_msg = f"""System Initialization Error
+
+The system failed to initialize with the following error:
+
+```
+{init_error}
+```
+
+Please check your configuration and environment variables."""
+        await cl.Message(content=error_msg).send()
+
+@cl.on_message
+async def main(message: cl.Message):
+    """Handle incoming messages"""
+    user_message = message.content
+    
+    # Get response from LLM with RAG steps shown FIRST
+    response_text, elapsed_time = await get_llm_response(user_message)
+    # logger.info(f"LLM Response:\n{response_text[:200]}...")  # Log first 200 chars
+
+    thinking_content, cleaned_response = extract_thinking_section(response_text)
+    
+    # Add timing stat to response
+    full_response = cleaned_response + f"\n\n⏱️ *Total time: {elapsed_time:.1f} seconds*"
+    
+    # THEN create a new message for streaming
+    msg = cl.Message(content="")
+    await msg.send()
+    
+    # Stream the response character by character for better UX
+    # This simulates streaming - in a real implementation you'd stream from the LLM
+    for i in range(0, len(full_response), 5):  # Stream in chunks of 5 characters
+        await msg.stream_token(full_response[i:i+5])
+        await asyncio.sleep(0.01)  # Small delay for visual effect
+    
+    # Update the final message
+    msg.content = full_response
+    await msg.update()
+
+## -------
+if __name__ == '__main__':
+    logger.info("App starting up...")
+    print(f"{'='*60}\n")

app_chainlit_graph.py

@@ -0,0 +1,375 @@
+"""
+GraphRAG Chainlit Application
+"""
+
+import chainlit as cl
+import os
+import logging
+from dotenv import load_dotenv
+import time
+import asyncio
+import re
+from typing import Dict, Any, Tuple
+
+# Apply nest_asyncio to allow nested event loops
+import nest_asyncio
+nest_asyncio.apply()
+
+# Import Step 1 functionality from setup module
+from query_graph_functions.setup import create_graphrag_setup
+# Import Steps 3-5 functionality from query preprocessing module
+from query_graph_functions.query_preprocessing import create_query_preprocessor, preprocess_query_pipeline
+# Import Steps 6-8 functionality from knowledge retrieval module
+from query_graph_functions.knowledge_retrieval import CommunitySearchEngine
+# Import Steps 9-12 functionality from follow-up search module
+from query_graph_functions.follow_up_search import FollowUpSearch
+# Import Steps 13-14 functionality from vector augmentation module
+from query_graph_functions.vector_augmentation import VectorAugmentationEngine
+# Import Steps 15-16 functionality from answer synthesis module
+from query_graph_functions.answer_synthesis import AnswerSynthesisEngine
+# Import Steps 17-20 functionality from response management module
+from query_graph_functions.response_management import ResponseManager
+from my_config import MY_CONFIG
+import query_utils
+
+# Configure environment
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+# Load environment variables
+load_dotenv()
+
+# Create logs directory if it doesn't exist
+os.makedirs('logs/chainlit', exist_ok=True)
+
+# Configure logging - Save to file and console
+logging.basicConfig(
+    level=logging.INFO, 
+    format='%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s',
+    handlers=[
+        logging.FileHandler('logs/chainlit/chainlit_graph.log', mode='a'),
+        logging.StreamHandler()
+    ],
+    force=True
+)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Log session start
+logger.info("=" * 80)
+logger.info(f"Chainlit GraphRAG Session Started - {time.strftime('%Y-%m-%d %H:%M:%S')}")
+logger.info("=" * 80)
+
+# Global GraphRAG engine instance
+graph_engine = None
+initialization_complete = False
+
+def initialize():
+    global graph_engine, initialization_complete
+    
+    if initialization_complete:
+        return
+    
+    logger.info("Initializing GraphRAG system...")
+    
+    try:
+        # Initialize setup module (Step 1)
+        setup = create_graphrag_setup()
+        
+        # Create GraphRAG engine wrapper
+        class GraphQueryEngine:
+            def __init__(self, setup):
+                self.setup = setup
+                self.neo4j_conn = setup.neo4j_conn
+                self.query_engine = setup.query_engine
+                self.graph_stats = setup.graph_stats
+                self.drift_config = setup.drift_config
+                self.llm = setup.llm
+                self.config = setup.config
+                self.query_preprocessor = None
+                self.response_manager = ResponseManager(setup)
+            
+            async def run_query_async(self, user_query):
+                """Execute GraphRAG query pipeline (Steps 3-20)"""
+                start_time = time.time()
+                
+                optimized_query = query_utils.tweak_query(user_query, MY_CONFIG.LLM_MODEL)
+                
+                try:
+                    if not self.setup.validate_system_readiness():
+                        return {"answer": "System not ready", "metadata": {}}
+                    
+                    # Initialize query preprocessor if needed
+                    if not self.query_preprocessor:
+                        self.query_preprocessor = await create_query_preprocessor(
+                            self.config, self.graph_stats
+                        )
+                    
+                    # Phase B: Query Preprocessing (Steps 3-5)
+                    analysis, routing, vectorization = await preprocess_query_pipeline(
+                        optimized_query, self.config, self.graph_stats
+                    )
+                    
+                    # Phase C: Community Retrieval (Steps 6-7)
+                    community_engine = CommunitySearchEngine(self.setup)
+                    community_results = await community_engine.execute_primer_phase(
+                        vectorization.embedding, routing
+                    )
+                    
+                    # Phase D: Follow-up Search (Steps 9-12)
+                    follow_up_engine = FollowUpSearch(self.setup)
+                    follow_up_results = await follow_up_engine.execute_follow_up_phase(
+                        community_results, routing
+                    )
+                    
+                    # Phase E: Vector Search Augmentation (Steps 13-14)
+                    vector_engine = VectorAugmentationEngine(self.setup)
+                    augmentation_results = await vector_engine.execute_vector_augmentation_phase(
+                        vectorization.embedding,
+                        {'communities': community_results['communities'], 
+                         'initial_answer': community_results['initial_answer'], 
+                         'follow_up_results': follow_up_results},
+                        routing
+                    )
+                    
+                    # Phase F: Answer Synthesis (Steps 15-16)
+                    synthesis_engine = AnswerSynthesisEngine(self.setup)
+                    synthesis_results = await synthesis_engine.execute_answer_synthesis_phase(
+                        analysis, routing, community_results, follow_up_results, augmentation_results
+                    )
+                    
+                    total_time = time.time() - start_time
+                    
+                    # Generate metadata
+                    metadata = self.response_manager.generate_comprehensive_metadata(
+                        analysis=analysis,
+                        routing=routing,
+                        vectorization=vectorization,
+                        community_results=community_results,
+                        follow_up_results=follow_up_results,
+                        augmentation_results=augmentation_results,
+                        synthesis_results=synthesis_results,
+                        total_time=total_time
+                    )
+                    
+                    # Cleanup async tasks
+                    await self.setup.cleanup_async_tasks(timeout=2.0)
+                    
+                    return {
+                        "answer": synthesis_results.final_answer,
+                        "metadata": metadata,
+                        "analysis": analysis,
+                        "routing": routing,
+                        "community_results": community_results,
+                        "follow_up_results": follow_up_results,
+                        "augmentation_results": augmentation_results
+                    }
+                    
+                except Exception as e:
+                    logger.error(f"Query pipeline error: {e}")
+                    synthesis_engine = AnswerSynthesisEngine(self.setup)
+                    return synthesis_engine.generate_error_response(f"Query error: {e}")
+        
+        graph_engine = GraphQueryEngine(setup)
+        
+        logger.info("✅ GraphRAG system initialized")
+        logger.info(f"✅ Using LLM: {MY_CONFIG.LLM_MODEL}")
+        logger.info(f"✅ Using embedding: {MY_CONFIG.EMBEDDING_MODEL}")
+        
+        initialization_complete = True
+        
+    except Exception as e:
+        initialization_complete = False
+        logger.error(f"GraphRAG initialization error: {str(e)}")
+        raise
+
+def extract_thinking_section(response_text):
+    """
+    Extract thinking section from LLM response if present.
+    
+    Args:
+        response_text (str): The full response from the LLM
+        
+    Returns:
+        tuple: (thinking_content, cleaned_response)
+            - thinking_content: Content within <think></think> tags or None if not found
+            - cleaned_response: Response with thinking section removed
+    """
+    thinking_pattern = r'<think>(.*?)</think>'
+    match = re.search(thinking_pattern, response_text, re.DOTALL)
+    
+    if match:
+        thinking_content = match.group(1).strip()
+        cleaned_response = re.sub(thinking_pattern, '', response_text, flags=re.DOTALL).strip()
+        return thinking_content, cleaned_response
+    else:
+        return None, response_text
+
+async def get_llm_response(message):
+    """
+    Process user message
+    """
+    global graph_engine, initialization_complete
+    
+    if not initialization_complete or graph_engine is None:
+        return "System not initialized. Please try again later.", 0
+    
+    start_time = time.time()
+    
+    try:
+        # Step 1: Query Preprocessing
+        async with cl.Step(name="Query Analysis", type="tool") as step:
+            step.input = message
+            optimized_query = query_utils.tweak_query(message, MY_CONFIG.LLM_MODEL)
+            step.output = f"Optimized query: {optimized_query}"
+        
+        # Execute GraphRAG query pipeline
+        result = await graph_engine.run_query_async(message)
+        
+        # Step 2: Community Search
+        if 'community_results' in result:
+            async with cl.Step(name="Community Retrieval", type="retrieval") as step:
+                communities = result['community_results'].get('communities', [])
+                step.input = "Searching graph communities"
+                step.output = f"Found {len(communities)} relevant communities"
+        
+        # Step 3: Follow-up Search
+        if 'follow_up_results' in result:
+            async with cl.Step(name="Entity Search", type="retrieval") as step:
+                step.input = "Analyzing entity relationships"
+                follow_up = result['follow_up_results']
+    
+                entities_found = len(follow_up.get('detailed_entities', []))
+                relationships_found = sum(~
+                    len(search.traversed_relationships) 
+                    for search in follow_up.get('local_search_results', [])
+                )
+                step.output = f"Entities: {entities_found}, Relationships: {relationships_found}"
+        
+        # Step 4: Vector Augmentation
+        if 'augmentation_results' in result:
+            async with cl.Step(name="Document Augmentation", type="retrieval") as step:
+                step.input = "Enriching with vector search"
+                aug_results = result['augmentation_results']
+                
+                if hasattr(aug_results, 'vector_results'):
+                    chunks = aug_results.vector_results
+                    step.output = f"Retrieved {len(chunks)} relevant document chunks"
+                else:
+                    step.output = "Vector augmentation completed"
+        
+        # Extract answer and timing
+        full_response = result.get('answer', 'No response generated')
+        
+        # Filter out metadata section more robustly
+        lines = full_response.split('\n')
+        filtered_lines = []
+        
+        for line in lines:
+            stripped = line.strip()
+            # Skip these metadata lines completely
+            if (stripped.startswith('## Comprehensive Answer') or 
+                stripped.startswith('# Comprehensive Answer') or
+                stripped.startswith('---') or
+                stripped.startswith('**Answer Confidence**:') or
+                stripped.startswith('**Sources Integrated**:') or
+                stripped.startswith('**Multi-Phase Coverage**:')):
+                continue
+            
+            filtered_lines.append(line)
+        
+        response_text = '\n'.join(filtered_lines).strip()
+        
+        # Extract thinking section if present
+        thinking_content, cleaned_response = extract_thinking_section(response_text)
+        
+        # Step 5: Optional Thinking Process
+        if thinking_content:
+            async with cl.Step(name="Reasoning Process", type="run") as step:
+                step.input = ""
+                step.output = thinking_content
+                logger.info(f"Thinking:\n{thinking_content[:200]}...")
+        
+        # Step 6: Final Answer
+        async with cl.Step(name="Synthesis", type="llm") as step:
+            step.input = "Generating comprehensive answer"
+            step.output = cleaned_response if cleaned_response else response_text
+        
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+        
+        return cleaned_response if cleaned_response else response_text, elapsed_time
+        
+    except Exception as e:
+        logger.error(f"Error processing query: {str(e)}")
+        return f"Sorry, I encountered an error:\n{str(e)}", 0
+
+# ====== CHAINLIT SPECIFIC CODE ======
+
+@cl.set_starters
+async def set_starters():
+    starters = []
+    for prompt in MY_CONFIG.STARTER_PROMPTS:
+        starters.append(
+            cl.Starter(
+                label=prompt.strip(),
+                message=prompt.strip(),
+            )
+        )
+    return starters
+## --- end: def set_starters(): ---
+
+@cl.on_chat_start
+async def start():
+    """Initialize the chat session"""
+    # Store initialization state in user session
+    cl.user_session.set("chat_started", True)
+    logger.info("User chat session started")
+    init_error = None
+    
+    try:
+        initialize()
+        # await cl.Message(content="How can I assist you today?").send()
+    except Exception as e:
+        init_error = str(e)
+        error_msg = f"""System Initialization Error
+
+The system failed to initialize with the following error:
+
+```
+{init_error}
+```
+
+Please check your configuration and environment variables."""
+        await cl.Message(content=error_msg).send()
+
+@cl.on_message
+async def main(message: cl.Message):
+    """Handle incoming messages"""
+    user_message = message.content
+    
+    # Get response from LLM with RAG steps shown FIRST
+    response_text, elapsed_time = await get_llm_response(user_message)
+    # logger.info(f"LLM Response:\n{response_text[:200]}...")  # Log first 200 chars
+
+    thinking_content, cleaned_response = extract_thinking_section(response_text)
+    
+    # Add timing stat to response
+    full_response = cleaned_response + f"\n\n⏱️ *Total time: {elapsed_time:.1f} seconds*"
+    
+    # THEN create a new message for streaming
+    msg = cl.Message(content="")
+    await msg.send()
+    
+    # Stream the response character by character for better UX
+    # This simulates streaming - in a real implementation you'd stream from the LLM
+    for i in range(0, len(full_response), 5):  # Stream in chunks of 5 characters
+        await msg.stream_token(full_response[i:i+5])
+        await asyncio.sleep(0.01)  # Small delay for visual effect
+    
+    # Update the final message
+    msg.content = full_response
+    await msg.update()
+
+## -------
+if __name__ == '__main__':
+    logger.info("App starting up...")

app_flask.py

@@ -0,0 +1,189 @@
+from flask import Flask, g, render_template, request, jsonify
+import os
+import logging
+import time
+
+# Import llama-index and related libraries
+from llama_index.core import VectorStoreIndex, StorageContext
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.core import Settings
+from llama_index.llms.litellm import LiteLLM
+from my_config import MY_CONFIG
+import query_utils
+
+
+os.environ['HF_ENDPOINT'] = MY_CONFIG.HF_ENDPOINT
+
+
+
+app = Flask(__name__)
+
+# Global variables for LLM and index
+vector_index = None
+
+initialization_complete = False
+def initialize():
+    """
+    Initialize LLM and Milvus vector database using llama-index.
+    This function sets up the necessary components for the chat application.
+    """
+    global vector_index, initialization_complete
+    
+    if initialization_complete:
+        return
+    
+    logging.info("Initializing LLM and vector database...")
+    
+    # raise Exception ("init exception test") # debug
+    
+    try:
+        ## embedding model
+        Settings.embed_model = HuggingFaceEmbedding(
+            model_name = MY_CONFIG.EMBEDDING_MODEL
+        )
+        print("✅ Using embedding model: ", MY_CONFIG.EMBEDDING_MODEL)
+        
+        # Setup LLM using LiteLLM
+        llm = LiteLLM(
+            model=MY_CONFIG.LLM_MODEL,
+            temperature=0.1
+        )
+        print("✅ LLM run environment: ", MY_CONFIG.LLM_RUN_ENV)
+        print("✅ Using LLM model : ", MY_CONFIG.LLM_MODEL)
+        Settings.llm = llm
+        
+        # Initialize Milvus vector store for Vector RAG only
+        vector_store = MilvusVectorStore(
+            uri = MY_CONFIG.MILVUS_URI_VECTOR ,  # Use dedicated Vector-only database
+            dim = MY_CONFIG.EMBEDDING_LENGTH , 
+            collection_name = MY_CONFIG.COLLECTION_NAME,
+            overwrite=False  # so we load the index from db
+        )
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        print ("✅ Connected to Vector-only Milvus instance: ", MY_CONFIG.MILVUS_URI_VECTOR )
+        
+        vector_index = VectorStoreIndex.from_vector_store(
+            vector_store=vector_store, storage_context=storage_context)
+        print ("✅ Loaded Vector-only index from:", MY_CONFIG.MILVUS_URI_VECTOR )
+
+        logging.info("Successfully initialized LLM and vector database")
+    
+        initialization_complete = True
+    except Exception as e:
+        initialization_complete = False
+        logging.error(f"Error initializing LLM and vector database: {str(e)}")
+        raise (e)
+        # return False
+## -------------
+
+## ----
+@app.route('/')
+def index():
+    init_error = app.config.get('INIT_ERROR', '')
+    # init_error = g.get('init_error', None)
+    return render_template('index.html', init_error=init_error)
+## end --- def index():
+
+
+## -----
+@app.route('/chat', methods=['POST'])
+def chat():
+    user_message = request.json.get('message')
+    
+    
+    # Get response from LLM
+    response = get_llm_response(user_message)
+    # print (response)
+    
+    return jsonify({'response': response})
+## end : def chat():
+
+
+def get_llm_response(message):
+    """
+    Process the user message and get a response from the LLM using Vector RAG
+    with structured prompting
+    """
+    global vector_index, initialization_complete
+    
+    # Check if LLM and index are initialized
+    if vector_index is None or  initialization_complete is None:
+        return "System did not initialize. Please try again later."
+    
+    start_time = time.time()
+    response_text = ''
+    
+    try:
+        # raise Exception ("chat exception test") ## debug
+        # Create a query engine from the index
+        query_engine = vector_index.as_query_engine()
+        
+        # Apply query optimization
+        message = query_utils.tweak_query(message, MY_CONFIG.LLM_MODEL)
+        
+        # Get initial vector response
+        vector_response = query_engine.query(message)
+        vector_text = str(vector_response).strip()
+        
+        # Structured prompt
+        structured_prompt = f"""Please provide a comprehensive, well-structured answer using the provided document information.
+
+Question: {message}
+
+Document Information:
+{vector_text}
+
+Instructions:
+1. Provide accurate, factual information based on the documents
+2. Structure your response clearly with proper formatting
+3. Be comprehensive yet concise
+4. Highlight key relationships and important details when relevant
+5. Use bullet points or sections when appropriate for clarity
+
+Please provide your answer:"""
+        
+        # Use structured prompt for final synthesis
+        final_response = query_engine.query(structured_prompt)
+        
+        if final_response:
+            response_text = str(final_response).strip()
+        
+    except Exception as e:
+        logging.error(f"Error getting LLM response: {str(e)}")
+        response_text =  f"Sorry, I encountered an error while processing your request:\n{str(e)}"
+        
+    end_time = time.time()
+    
+    # add timing stat
+    response_text += f"\n⏱️ *Total time: {(end_time - start_time):.1f} seconds*"
+    return response_text
+    
+## --- end: def get_llm_response():
+
+
+    
+
+## -------
+if __name__ == '__main__':
+    # Configure logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+    logging.info("App starting up...")
+    
+    # Initialize LLM and vector database
+    try:
+        initialize()
+    except Exception as e:
+        logging.warning("Starting without LLM and vector database. Responses will be limited.")
+        app.config['INIT_ERROR'] = str(e)
+        # g.init_error = str(e)
+        
+    
+    # Vector RAG Flask App - Configurable port via environment
+    PORT = MY_CONFIG.FLASK_VECTOR_PORT
+    print(f"🚀 Vector RAG Flask app starting on port {PORT}")
+    app.run(host="0.0.0.0", debug=False, port=PORT)
+## -- end main ----

app_flask_graph.py

@@ -0,0 +1,264 @@
+"""
+GraphRAG Flask Web Application
+"""
+
+from flask import Flask, render_template, request, jsonify
+import os
+import logging
+import time
+import asyncio
+import nest_asyncio
+from query_graph_functions.setup import create_graphrag_setup
+from query_graph_functions.query_preprocessing import create_query_preprocessor, preprocess_query_pipeline
+from query_graph_functions.knowledge_retrieval import CommunitySearchEngine
+from query_graph_functions.follow_up_search import FollowUpSearch
+from query_graph_functions.vector_augmentation import VectorAugmentationEngine
+from query_graph_functions.answer_synthesis import AnswerSynthesisEngine
+from query_graph_functions.response_management import ResponseManager
+from my_config import MY_CONFIG
+import query_utils
+
+
+nest_asyncio.apply()
+os.environ['HF_ENDPOINT'] = MY_CONFIG.HF_ENDPOINT
+
+app = Flask(__name__)
+
+# Global GraphRAG engine
+graph_engine = None
+initialization_complete = False
+
+def initialize():
+    """
+    Initialize GraphRAG system
+    """
+    global graph_engine, initialization_complete
+    
+    if initialization_complete:
+        return
+    
+    logging.info("Initializing GraphRAG system...")
+    
+    try:
+        # Initialize setup module (Step 1)
+        setup = create_graphrag_setup()
+        
+        # Create GraphRAG engine wrapper
+        class GraphQueryEngine:
+            def __init__(self, setup):
+                self.setup = setup
+                self.neo4j_conn = setup.neo4j_conn
+                self.query_engine = setup.query_engine
+                self.graph_stats = setup.graph_stats
+                self.drift_config = setup.drift_config
+                self.llm = setup.llm
+                self.config = setup.config
+                self.query_preprocessor = None
+                self.response_manager = ResponseManager(setup)
+            
+            async def run_query_async(self, user_query):
+                """Execute GraphRAG query pipeline (Steps 3-20)"""
+                start_time = time.time()
+                
+                optimized_query = query_utils.tweak_query(user_query, MY_CONFIG.LLM_MODEL)
+                
+                try:
+                    if not self.setup.validate_system_readiness():
+                        return {"answer": "System not ready", "metadata": {}}
+                    
+                    # Initialize query preprocessor if needed
+                    if not self.query_preprocessor:
+                        self.query_preprocessor = await create_query_preprocessor(
+                            self.config, self.graph_stats
+                        )
+                    
+                    # Phase B: Query Preprocessing (Steps 3-5)
+                    analysis, routing, vectorization = await preprocess_query_pipeline(
+                        optimized_query, self.config, self.graph_stats
+                    )
+                    
+                    # Phase C: Community Retrieval (Steps 6-7)
+                    community_engine = CommunitySearchEngine(self.setup)
+                    community_results = await community_engine.execute_primer_phase(
+                        vectorization.embedding, routing
+                    )
+                    
+                    # Phase D: Follow-up Search (Steps 9-12)
+                    follow_up_engine = FollowUpSearch(self.setup)
+                    follow_up_results = await follow_up_engine.execute_follow_up_phase(
+                        community_results, routing
+                    )
+                    
+                    # Phase E: Vector Search Augmentation (Steps 13-14)
+                    vector_engine = VectorAugmentationEngine(self.setup)
+                    augmentation_results = await vector_engine.execute_vector_augmentation_phase(
+                        vectorization.embedding,
+                        {'communities': community_results['communities'], 
+                         'initial_answer': community_results['initial_answer'], 
+                         'follow_up_results': follow_up_results},
+                        routing
+                    )
+                    
+                    # Phase F: Answer Synthesis (Steps 15-16)
+                    synthesis_engine = AnswerSynthesisEngine(self.setup)
+                    synthesis_results = await synthesis_engine.execute_answer_synthesis_phase(
+                        analysis, routing, community_results, follow_up_results, augmentation_results
+                    )
+                    
+                    total_time = time.time() - start_time
+                    
+                    # Generate metadata
+                    metadata = self.response_manager.generate_comprehensive_metadata(
+                        analysis=analysis,
+                        routing=routing,
+                        vectorization=vectorization,
+                        community_results=community_results,
+                        follow_up_results=follow_up_results,
+                        augmentation_results=augmentation_results,
+                        synthesis_results=synthesis_results,
+                        total_time=total_time
+                    )
+                    
+                    # Cleanup async tasks
+                    await self.setup.cleanup_async_tasks(timeout=2.0)
+                    
+                    return {
+                        "answer": synthesis_results.final_answer,
+                        "metadata": metadata
+                    }
+                    
+                except Exception as e:
+                    logging.error(f"Query pipeline error: {e}")
+                    synthesis_engine = AnswerSynthesisEngine(self.setup)
+                    return synthesis_engine.generate_error_response(f"Query error: {e}")
+            
+            def run_query(self, user_query):
+                """Synchronous wrapper for async query"""
+                try:
+                    loop = asyncio.get_event_loop()
+                    return loop.run_until_complete(self.run_query_async(user_query))
+                except Exception as e:
+                    logging.error(f"Query execution error: {e}")
+                    return {"answer": f"Error: {e}", "metadata": {}}
+        
+        graph_engine = GraphQueryEngine(setup)
+        
+        print("✅ GraphRAG system initialized")
+        print(f"✅ Using LLM: {MY_CONFIG.LLM_MODEL}")
+        print(f"✅ Using embedding: {MY_CONFIG.EMBEDDING_MODEL}")
+        
+        logging.info("GraphRAG system ready")
+        initialization_complete = True
+        
+    except Exception as e:
+        initialization_complete = False
+        logging.error(f"GraphRAG initialization error: {str(e)}")
+        raise
+
+## ----
+@app.route('/')
+def index():
+    init_error = app.config.get('INIT_ERROR', '')
+    # init_error = g.get('init_error', None)
+    return render_template('index.html', init_error=init_error)
+## end --- def index():
+
+
+## ----
+@app.route('/health')
+def health():
+    """Health check endpoint for deployment platforms"""
+    if initialization_complete:
+        return jsonify({"status": "healthy", "graphrag": "initialized"}), 200
+    else:
+        return jsonify({"status": "initializing"}), 503
+## end --- def health():
+
+
+## -----
+@app.route('/chat', methods=['POST'])
+def chat():
+    user_message = request.json.get('message')
+    
+    
+    # Get response from LLM
+    response = get_llm_response(user_message)
+    # print (response)
+    
+    return jsonify({'response': response})
+## end : def chat():
+
+
+def get_llm_response(message):
+    """
+    Process user message using complete GraphRAG pipeline.
+    Implements the full 25-step DRIFT search methodology.
+    """
+    global graph_engine, initialization_complete
+    
+    if not initialization_complete or graph_engine is None:
+        return "System not initialized. Please try again later."
+    
+    start_time = time.time()
+    
+    try:
+        # Execute GraphRAG query pipeline
+        result = graph_engine.run_query(message)
+        
+        # Extract answer and timing
+        full_response = result.get('answer', 'No response generated')
+        
+        # Filter out metadata section more robustly
+        lines = full_response.split('\n')
+        filtered_lines = []
+        
+        for line in lines:
+            stripped = line.strip()
+            # Skip these metadata lines completely
+            if (stripped.startswith('## Comprehensive Answer') or 
+                stripped.startswith('# Comprehensive Answer') or
+                stripped.startswith('---') or
+                stripped.startswith('**Answer Confidence**:') or
+                stripped.startswith('**Sources Integrated**:') or
+                stripped.startswith('**Multi-Phase Coverage**:')):
+                continue
+            
+            filtered_lines.append(line)
+        
+        response_text = '\n'.join(filtered_lines).strip()
+        end_time = time.time()
+        
+        # Add timing information
+        response_text += f"\n\n⏱️ *Total time: {(end_time - start_time):.1f} seconds*"
+        
+        return response_text
+        
+    except Exception as e:
+        logging.error(f"Error processing query: {str(e)}")
+        return f"Sorry, I encountered an error:\n{str(e)}"
+
+
+    
+
+## -------
+if __name__ == '__main__':
+    # Configure logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+    logging.info("App starting up...")
+    
+    # Initialize LLM and vector database
+    try:
+        initialize()
+    except Exception as e:
+        logging.warning("Starting without LLM and vector database. Responses will be limited.")
+        app.config['INIT_ERROR'] = str(e)
+        # g.init_error = str(e)
+        
+    
+    # GraphRAG Flask App - Configurable port via environment
+    PORT = MY_CONFIG.FLASK_GRAPH_PORT
+    print(f"🚀 GraphRAG Flask app starting on port {PORT}")
+    app.run(host="0.0.0.0", debug=False, port=PORT)

chainlit.md

@@ -0,0 +1,14 @@
+# Welcome to Chainlit! 🚀🤖
+
+Hi there, Developer! 👋 We're excited to have you on board. Chainlit is a powerful tool designed to help you prototype, debug and share applications built on top of LLMs.
+
+## Useful Links 🔗
+
+- **Documentation:** Get started with our comprehensive [Chainlit Documentation](https://docs.chainlit.io) 📚
+- **Discord Community:** Join our friendly [Chainlit Discord](https://discord.gg/k73SQ3FyUh) to ask questions, share your projects, and connect with other developers! 💬
+
+We can't wait to see what you create with Chainlit! Happy coding! 💻😊
+
+## Welcome screen
+
+To modify the welcome screen, edit the `chainlit.md` file at the root of your project. If you do not want a welcome screen, just leave this file empty.

cleanup_pipeline_deps.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# ============================================
+# Cleanup Pipeline Dependencies Script
+# ============================================
+# This script removes heavy packages that are only needed for the pipeline
+# to save ~350-500 MB of RAM in production Docker containers.
+#
+# Run this AFTER the pipeline completes successfully.
+# ============================================
+
+echo "============================================"
+echo "Starting Pipeline Dependency Cleanup"
+echo "============================================"
+echo "This will remove packages only needed for:"
+echo "  - Document processing (docling, html2text)"
+echo "  - Graph community detection (igraph, leidenalg, etc.)"
+echo "  - Development tools (ipykernel, tqdm, etc.)"
+echo ""
+echo "Estimated RAM savings: 350-500 MB"
+echo "============================================"
+
+# Document processing packages
+echo "Removing document processing packages..."
+pip uninstall -y docling html2text 2>/dev/null || echo "  (already removed or not installed)"
+
+# Graph community detection packages
+echo "Removing graph community detection packages..."
+pip uninstall -y python-louvain igraph leidenalg graspologic 2>/dev/null || echo "  (already removed or not installed)"
+
+# Development tools
+echo "Removing development tools..."
+pip uninstall -y tqdm ipykernel fastmcp 2>/dev/null || echo "  (already removed or not installed)"
+
+# Milvus Lite (if using cloud Zilliz)
+if [ "$VECTOR_DB_TYPE" = "cloud_zilliz" ]; then
+    echo "Removing Milvus Lite (using cloud Zilliz)..."
+    pip uninstall -y milvus-lite 2>/dev/null || echo "  (already removed or not installed)"
+fi
+
+# Chainlit (if using Flask only)
+if [ "$APP_TYPE" = "flask_graph" ] || [ "$APP_TYPE" = "flask" ]; then
+    echo "Removing Chainlit (using Flask app)..."
+    pip uninstall -y chainlit 2>/dev/null || echo "  (already removed or not installed)"
+fi
+
+echo ""
+echo "============================================"
+echo "Cleanup Complete!"
+echo "============================================"
+echo "Before cleanup: ~800 MB"
+echo "After cleanup:  ~300-450 MB (depending on config)"
+echo ""
+echo "Note: If you redeploy and AUTO_RUN_PIPELINE=true,"
+echo "      all packages will be reinstalled automatically."
+echo "============================================"

docker-compose.cloud.yml

@@ -0,0 +1,34 @@
+version: '3.8'
+
+services:
+  allycat-graphrag-cloud:
+    image: allycat-graphrag:cloud
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        INSTALL_OLLAMA: "false"
+        INSTALL_LOCAL_VECTOR_DB: "false"
+    container_name: allycat-cloud
+    ports:
+      - "${DOCKER_PORT:-8080}:${DOCKER_APP_PORT:-8080}"
+    environment:
+      - LLM_RUN_ENV=cloud
+      - VECTOR_DB_TYPE=cloud_zilliz
+      - APP_TYPE=${APP_TYPE:-flask_graph}
+      - AUTO_RUN_PIPELINE=${AUTO_RUN_PIPELINE:-false}
+      - DOCKER_APP_PORT=${DOCKER_APP_PORT:-8080}
+      - FLASK_GRAPH_PORT=${FLASK_GRAPH_PORT:-8080}
+      - FLASK_VECTOR_PORT=${FLASK_VECTOR_PORT:-8081}
+      - CHAINLIT_GRAPH_PORT=${CHAINLIT_GRAPH_PORT:-8083}
+      - CHAINLIT_VECTOR_PORT=${CHAINLIT_VECTOR_PORT:-8082}
+    env_file:
+      - .env
+    command: ["deploy"]
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:${DOCKER_APP_PORT:-8080}"]
+      interval: 60s
+      timeout: 60s
+      retries: 1
+      start_period: 1500s

docker-compose.hybrid.yml

@@ -0,0 +1,36 @@
+version: '3.8'
+
+services:
+  allycat-graphrag-hybrid:
+    image: allycat-graphrag:hybrid
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        INSTALL_OLLAMA: "false"
+        INSTALL_LOCAL_VECTOR_DB: "true"
+    container_name: allycat-hybrid
+    ports:
+      - "${DOCKER_PORT:-8080}:${DOCKER_APP_PORT:-8080}"
+    environment:
+      - LLM_RUN_ENV=cloud
+      - VECTOR_DB_TYPE=local
+      - APP_TYPE=${APP_TYPE:-flask_graph}
+      - AUTO_RUN_PIPELINE=${AUTO_RUN_PIPELINE:-false}
+      - DOCKER_APP_PORT=${DOCKER_APP_PORT:-8080}
+      - FLASK_GRAPH_PORT=${FLASK_GRAPH_PORT:-8080}
+      - FLASK_VECTOR_PORT=${FLASK_VECTOR_PORT:-8081}
+      - CHAINLIT_GRAPH_PORT=${CHAINLIT_GRAPH_PORT:-8083}
+      - CHAINLIT_VECTOR_PORT=${CHAINLIT_VECTOR_PORT:-8082}
+    env_file:
+      - .env
+    volumes:
+      - ./workspace:/allycat/workspace
+    command: ["deploy"]
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:${DOCKER_APP_PORT:-8080}"]
+      interval: 60s
+      timeout: 60s
+      retries: 1
+      start_period: 1500s

docker-compose.local.yml

@@ -0,0 +1,38 @@
+version: '3.8'
+
+services:
+  allycat-graphrag-local:
+    image: allycat-graphrag:local
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        INSTALL_OLLAMA: "true"
+        INSTALL_LOCAL_VECTOR_DB: "true"
+    container_name: allycat-local
+    ports:
+      - "${DOCKER_PORT:-8080}:${DOCKER_APP_PORT:-8080}"
+      - "${OLLAMA_PORT:-11434}:11434"
+    environment:
+      - LLM_RUN_ENV=local_ollama
+      - VECTOR_DB_TYPE=local
+      - APP_TYPE=${APP_TYPE:-flask_graph}
+      - AUTO_RUN_PIPELINE=${AUTO_RUN_PIPELINE:-false}
+      - DOCKER_APP_PORT=${DOCKER_APP_PORT:-8080}
+      - FLASK_GRAPH_PORT=${FLASK_GRAPH_PORT:-8080}
+      - FLASK_VECTOR_PORT=${FLASK_VECTOR_PORT:-8081}
+      - CHAINLIT_GRAPH_PORT=${CHAINLIT_GRAPH_PORT:-8083}
+      - CHAINLIT_VECTOR_PORT=${CHAINLIT_VECTOR_PORT:-8082}
+      - OLLAMA_PORT=${OLLAMA_PORT:-11434}
+    env_file:
+      - .env
+    volumes:
+      - ./workspace:/allycat/workspace
+    command: ["deploy"]
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:${DOCKER_APP_PORT:-8080}"]
+      interval: 60s
+      timeout: 60s
+      retries: 1
+      start_period: 1500s

docker-startup.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+
+echo "=== AllyCAT GraphRAG Docker Startup ==="
+
+# Check deployment mode from environment
+LLM_MODE=${LLM_RUN_ENV:-cloud}
+VECTOR_MODE=${VECTOR_DB_TYPE:-cloud_zilliz}
+
+echo "LLM Mode: $LLM_MODE"
+echo "Vector DB Mode: $VECTOR_MODE"
+
+# Conditional: Start Ollama only if in local mode
+if [ "$LLM_MODE" = "local_ollama" ]; then
+    echo "Starting Ollama in local mode..."
+    
+    # Define OLLAMA_MODELS dir
+    if [ -z "$OLLAMA_MODELS" ]; then
+        export OLLAMA_MODELS=/allycat/workspace/ollama
+    fi
+    
+    echo "Env variables for OLLAMA:"
+    env | grep OLLAMA
+    
+    # Start ollama
+    ollama_model=${OLLAMA_MODEL:-gemma3:1b}
+    echo "Starting Ollama server..."
+    ollama serve > /allycat/ollama-serve.out 2>&1 &
+    
+    # Wait for ollama to start
+    OLLAMA_PORT=${OLLAMA_PORT:-11434}
+    while ! nc -z localhost $OLLAMA_PORT; do
+        sleep 1
+    done
+    echo "✅ Ollama started on port $OLLAMA_PORT"
+    
+    # Only download the model if we are in DEPLOY mode
+    if [ "$1" == "deploy" ]; then
+        echo "Downloading Ollama model: $ollama_model"
+        ollama pull $ollama_model
+        echo "✅ Ollama model downloaded: $ollama_model"
+    fi
+else
+    echo "✅ Using cloud LLM mode - Ollama not started"
+fi
+
+# Conditional: Setup local vector DB only if needed
+if [ "$VECTOR_MODE" = "local" ]; then
+    echo "Setting up local Milvus vector database..."
+    mkdir -p /allycat/workspace
+    echo "✅ Local vector database directory created"
+else
+    echo "✅ Using Zilliz Cloud for vector database"
+fi
+
+# Run GraphRAG pipeline if AUTO_RUN_PIPELINE is enabled and in deploy mode
+if [ "$1" == "deploy" ] && [ "${AUTO_RUN_PIPELINE:-false}" = "true" ]; then
+    echo ""
+    echo "=== Running GraphRAG Pipeline Automatically ==="
+    echo ""
+    
+    # Step 1: Crawl website
+    if [ -n "$WEBSITE_URL" ]; then
+        echo "Step 1/5: Crawling website: $WEBSITE_URL"
+        python3 1_crawl_site.py || echo "⚠️  Warning: Crawl failed, continuing..."
+        echo "✅ Step 1 complete"
+        echo ""
+    else
+        echo "⚠️  Skipping crawl - WEBSITE_URL not set"
+    fi
+    
+    # Step 2: Process files to markdown
+    echo "Step 2/5: Processing files to markdown..."
+    python3 2_process_files.py || echo "⚠️  Warning: Processing failed, continuing..."
+    echo "✅ Step 2 complete"
+    echo ""
+    
+    # Step 3: Save to vector database
+    echo "Step 3/5: Saving to vector database..."
+    if [ "$VECTOR_MODE" = "cloud_zilliz" ]; then
+        python3 3_save_to_vector_db_zilliz.py || echo "⚠️  Warning: Vector DB save failed, continuing..."
+    else
+        python3 3_save_to_vector_db.py || echo "⚠️  Warning: Vector DB save failed, continuing..."
+    fi
+    echo "✅ Step 3 complete"
+    echo ""
+    
+    # Step 4: Process graph data (3 phases)
+    echo "Step 4/5: Processing graph data (3 phases)..."
+    echo "  Phase 1: Extracting entities and relationships..."
+    python3 2b_process_graph_phase1.py || echo "⚠️  Warning: Phase 1 failed, continuing..."
+    echo "  Phase 2: Building communities..."
+    python3 2b_process_graph_phase2.py || echo "⚠️  Warning: Phase 2 failed, continuing..."
+    echo "  Phase 3: Generating community summaries..."
+    python3 2b_process_graph_phase3.py || echo "⚠️  Warning: Phase 3 failed, continuing..."
+    echo "✅ Step 4 complete"
+    echo ""
+    
+    # Step 5: Save to graph database
+    echo "Step 5/5: Saving to graph database..."
+    python3 3b_save_to_graph_db.py || echo "⚠️  Warning: Graph DB save failed, continuing..."
+    echo "✅ Step 5 complete"
+    echo ""
+    
+    echo "=== ✅ Pipeline Complete - Starting Application ==="
+    echo ""
+    
+    # OPTIMIZATION: Clean up pipeline dependencies to save RAM
+    if [ "${CLEANUP_PIPELINE_DEPS:-false}" = "true" ]; then
+        echo ""
+        echo "=== 🧹 Cleaning Up Pipeline Dependencies ==="
+        echo "This will save ~350-500 MB of RAM"
+        echo ""
+        chmod +x ./cleanup_pipeline_deps.sh
+        ./cleanup_pipeline_deps.sh
+        echo ""
+        echo "=== ✅ Cleanup Complete ==="
+        echo ""
+    else
+        echo ""
+        echo "💡 TIP: Set CLEANUP_PIPELINE_DEPS=true in .env to save ~350-500 MB RAM"
+        echo "        after pipeline completes (reduces OOM errors on 1GB containers)"
+        echo ""
+    fi
+fi
+
+# Start the appropriate web application
+APP_TYPE=${APP_TYPE:-flask_graph}
+DOCKER_APP_PORT=${DOCKER_APP_PORT:-8080}
+FLASK_GRAPH_PORT=${FLASK_GRAPH_PORT:-8080}
+FLASK_VECTOR_PORT=${FLASK_VECTOR_PORT:-8081}
+CHAINLIT_GRAPH_PORT=${CHAINLIT_GRAPH_PORT:-8083}
+CHAINLIT_VECTOR_PORT=${CHAINLIT_VECTOR_PORT:-8082}
+
+# Log port configuration
+echo ""
+echo "=== Port Configuration ==="
+echo "DOCKER_APP_PORT (internal container): $DOCKER_APP_PORT"
+echo "FLASK_GRAPH_PORT: $FLASK_GRAPH_PORT"
+echo "FLASK_VECTOR_PORT: $FLASK_VECTOR_PORT"
+echo "CHAINLIT_GRAPH_PORT: $CHAINLIT_GRAPH_PORT"
+echo "CHAINLIT_VECTOR_PORT: $CHAINLIT_VECTOR_PORT"
+echo ""
+
+# Determine which port will be used based on APP_TYPE
+case $APP_TYPE in
+    "flask_graph")
+        APP_PORT=$FLASK_GRAPH_PORT
+        ;;
+    "chainlit_graph")
+        APP_PORT=$CHAINLIT_GRAPH_PORT
+        ;;
+    "flask")
+        APP_PORT=$FLASK_VECTOR_PORT
+        ;;
+    "chainlit")
+        APP_PORT=$CHAINLIT_VECTOR_PORT
+        ;;
+    *)
+        APP_PORT=$FLASK_GRAPH_PORT
+        ;;
+esac
+
+echo "Selected APP_TYPE: $APP_TYPE will run on port: $APP_PORT"
+echo "Container will expose application on port: $DOCKER_APP_PORT (mapped to host DOCKER_PORT)"
+echo ""
+
+if [ "$1" == "deploy" ]; then
+    echo "In deploy mode..."
+    
+    case $APP_TYPE in
+        "flask_graph")
+            echo "Starting Flask GraphRAG app on port $FLASK_GRAPH_PORT..."
+            python3 app_flask_graph.py
+            ;;
+        "chainlit_graph")
+            echo "Starting Chainlit GraphRAG app on port $CHAINLIT_GRAPH_PORT..."
+            chainlit run app_chainlit_graph.py --host 0.0.0.0 --port $CHAINLIT_GRAPH_PORT
+            ;;
+        "flask")
+            echo "Starting Flask Vector RAG app on port $FLASK_VECTOR_PORT..."
+            python3 app_flask.py
+            ;;
+        "chainlit")
+            echo "Starting Chainlit Vector RAG app on port $CHAINLIT_VECTOR_PORT..."
+            chainlit run app_chainlit.py --host 0.0.0.0 --port $CHAINLIT_VECTOR_PORT
+            ;;
+        *)
+            echo "Starting default Flask GraphRAG app on port $FLASK_GRAPH_PORT..."
+            python3 app_flask_graph.py
+            ;;
+    esac
+else
+    echo "Not in deploy mode, entering interactive shell."
+    echo ""
+    echo "Available commands:"
+    echo "  python3 app_flask_graph.py       - Start Flask GraphRAG app"
+    echo "  python3 app_flask.py             - Start Flask VectorRAG app"
+    echo "  chainlit run app_chainlit_graph.py - Start Chainlit GraphRAG app"
+    echo "  chainlit run app_chainlit.py     - Start Chainlit VectorRAG app"
+    
+    if [ "$LLM_MODE" = "local_ollama" ]; then
+        echo "  ollama pull $ollama_model        - Download Ollama model"
+    fi
+    echo ""
+    /bin/bash
+fi

env.sample.txt

@@ -0,0 +1,177 @@
+# ============================================
+# AllyCAT GraphRAG Configuration
+# ============================================
+# This file contains all configuration options for AllyCAT GraphRAG.
+# Copy this file to .env and customize the values.
+
+# ============================================
+# Deployment Mode
+# ============================================
+# Automatically run the complete pipeline on startup (Docker deployments)
+# Set to true for Heroku, AWS, Google Cloud Run, etc.
+AUTO_RUN_PIPELINE=false
+
+# Memory Optimization: Remove pipeline dependencies after completion
+# Saves ~350-500 MB RAM - recommended for 1GB containers (DigitalOcean, etc.)
+# Set to true to automatically clean up heavy packages after pipeline completes
+CLEANUP_PIPELINE_DEPS=false
+
+# ============================================
+# Website Crawling Configuration
+# ============================================
+# Website to crawl (required if AUTO_RUN_PIPELINE=true)
+WEBSITE_URL=https://example.com
+CRAWL_MAX_DOWNLOADS=100
+CRAWL_MAX_DEPTH=3
+WAITTIME_BETWEEN_REQUESTS=0.1
+
+# ============================================
+# LLM Configuration (Cloud-First)
+# ============================================
+# LLM Runtime Environment
+# Options: cloud, local_ollama
+LLM_RUN_ENV=cloud
+
+# LLM Model Selection
+# Cloud providers: cerebras/llama3.1-8b, gemini/gemini-1.5-flash, nebius/meta-llama/Meta-Llama-3.1-8B-Instruct
+# Local: ollama/gemma3:1b
+LLM_MODEL=cerebras/llama3.1-8b
+
+# ============================================
+# LLM API Keys (Cloud Providers)
+# ============================================
+# Get your FREE API keys:
+# - Cerebras: https://cerebras.ai/ (recommended)
+# - Gemini: https://aistudio.google.com/
+# - Nebius: https://studio.nebius.ai/
+
+CEREBRAS_API_KEY=your_cerebras_api_key
+GEMINI_API_KEY=your_gemini_api_key
+NEBIUS_API_KEY=your_nebius_api_key
+
+# ============================================
+# Local Ollama Configuration (Optional)
+# ============================================
+# Only needed if LLM_RUN_ENV=local_ollama
+# OLLAMA_MODEL=gemma3:1b
+# OLLAMA_BASE_URL=http://localhost:11434
+
+# ============================================
+# Vector Database Configuration
+# ============================================
+# Options: cloud_zilliz (recommended), local
+VECTOR_DB_TYPE=cloud_zilliz
+
+# Zilliz Cloud Configuration (https://cloud.zilliz.com/)
+ZILLIZ_CLUSTER_ENDPOINT=https://your-cluster.zilliz.cloud
+ZILLIZ_TOKEN=your_zilliz_token
+
+# Local Milvus Configuration (only if VECTOR_DB_TYPE=local)
+# MILVUS_URI=./workspace/milvus_lite.db
+
+# ============================================
+# Graph Database Configuration (Neo4j)
+# ============================================
+# Neo4j Aura (Cloud) - Recommended: https://neo4j.com/cloud/aura/
+NEO4J_URI=neo4j+s://your-instance.databases.neo4j.io
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD=your_neo4j_password
+NEO4J_DATABASE=neo4j
+
+# Local Neo4j (only for development)
+# NEO4J_URI=bolt://localhost:7687
+
+# ============================================
+# Graph Extraction LLM Provider
+# ============================================
+# Provider for entity/relationship extraction
+# Options: gemini (recommended, 1500 free requests/day), cerebras
+GRAPH_LLM_PROVIDER=gemini
+
+# API keys are shared from LLM Configuration section above
+
+# ============================================
+# Embedding Model Configuration
+# ============================================
+# Embedding model for semantic search
+# Options:
+#   - ibm-granite/granite-embedding-30m-english (61 MB, fastest)
+#   - BAAI/bge-small-en-v1.5 (129 MB, balanced)
+#   - ibm-granite/granite-embedding-107m-multilingual (219 MB, multilingual)
+EMBEDDING_MODEL=ibm-granite/granite-embedding-30m-english
+EMBEDDING_LENGTH=384
+
+# ============================================
+# Chunking Configuration
+# ============================================
+CHUNK_SIZE=512
+CHUNK_OVERLAP=20
+
+# ============================================
+# Graph Extraction Configuration
+# ============================================
+# Entity and relationship extraction parameters
+GRAPH_MIN_ENTITIES=5
+GRAPH_MAX_ENTITIES=15
+GRAPH_MIN_RELATIONSHIPS=3
+GRAPH_MAX_RELATIONSHIPS=8
+GRAPH_MIN_CONFIDENCE=0.8
+GRAPH_MAX_CONTENT_CHARS=12000
+GRAPH_SENTENCE_BOUNDARY_RATIO=0.7
+
+# ============================================
+# Graph Community Detection (Phase 2)
+# ============================================
+# Leiden algorithm parameters for community detection
+GRAPH_MIN_COMMUNITY_SIZE=5
+GRAPH_LEIDEN_RESOLUTION=1.0
+GRAPH_LEIDEN_ITERATIONS=-1
+GRAPH_LEIDEN_SEED=42
+GRAPH_TARGET_COVERAGE_MIN=5.0
+GRAPH_TARGET_COVERAGE_MAX=8.0
+GRAPH_RESOLUTION_CANDIDATES=0.1,0.5,1.0,2.0,5.0,10.0,20.0,30.0,50.0,100.0
+GRAPH_MIN_NODES_FOR_OPTIMIZATION=50
+
+# ============================================
+# Application Configuration
+# ============================================
+# Application type for Docker deployment
+# Options: flask_graph (default), chainlit_graph, flask
+APP_TYPE=flask_graph
+
+# Flask server port
+PORT=8080
+
+# UI starter prompts (pipe-separated)
+UI_STARTER_PROMPTS=What is this website?  |  What are upcoming events?  | Who are some of the partners?
+
+# ============================================
+# Port Configuration
+# ============================================
+# Flask apps (Vector RAG vs GraphRAG) - Auto-configured via MY_CONFIG
+FLASK_VECTOR_PORT=8081      # app_flask.py (vector-only RAG)
+FLASK_GRAPH_PORT=8080       # app_flask_graph.py (GraphRAG)
+
+# Chainlit apps (interactive UI) - Default port: 8000, custom ports for Docker
+CHAINLIT_VECTOR_PORT=8082   # app_chainlit.py (Docker only; native Python uses 8000)
+CHAINLIT_GRAPH_PORT=8083    # app_chainlit_graph.py (Docker only; native Python uses 8000)
+
+# Docker and external services
+DOCKER_PORT=8080            # External Docker exposed port (host side)
+DOCKER_APP_PORT=8080        # Internal container port (container side, matches APP_TYPE)
+OLLAMA_PORT=11434           # Ollama server port (for local LLM)
+
+# ============================================
+# Workspace Configuration
+# ============================================
+# For native execution: use relative path 'workspace'
+# For Docker: use absolute path '/allycat/workspace'
+WORKSPACE_DIR=workspace
+
+# ============================================
+# Advanced Configuration
+# ============================================
+# Hugging Face endpoint (for Chinese users or custom mirrors)
+HF_ENDPOINT=https://huggingface.co
+
+

file_utils.py

@@ -0,0 +1,56 @@
+import os
+import requests
+from humanfriendly import format_size
+import pandas as pd
+import glob
+from urllib.parse import unquote
+
+
+## Reads parquet files in a folder into a pandas dataframe
+def read_parquet_files_as_df (parquet_dir):
+    parquet_files = glob.glob(f'{parquet_dir}/*.parquet')
+
+    # read each parquet file into a DataFrame and store in a list
+    dfs = [pd.read_parquet (f) for f in parquet_files]
+
+    # Concatenate all DataFrames into a single DataFrame
+    data_df = pd.concat(dfs, ignore_index=True)
+    return data_df
+
+
+def download_file(url, local_file, chunk_size=1024*1024):
+    """
+    Downloads a remote URL to a local file.
+
+    Args:
+        url (str): The remote URL.
+        local_filename (str): The name of the local file to save the downloaded content.
+        chunk_size (int): The size in bytes of each chunk. Defaults to 1024.
+
+    Returns:
+        None
+        
+    Example usage:
+        download_file('http://example.com/file.txt', 'file.txt', chunk_size=1024*1024)  # Download in chunks of 1MB
+    """
+    # Check if the local file already exists
+    if os.path.exists(local_file):
+        file_size = format_size(os.path.getsize(local_file))
+        print(f"Local file '{local_file}' ({file_size}) already exists. Skipping download.")
+        return
+
+    # Create the directory if it doesn't exist
+    os.makedirs(os.path.dirname(local_file), exist_ok=True)
+
+    # Stream the file download
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_file, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=chunk_size):
+                if chunk: # filter out keep-alive new chunks
+                    f.write(chunk)
+        print()
+        file_size = format_size(os.path.getsize(local_file))
+        print(f"{local_file} ({file_size}) downloaded successfully.")
+## --- end: download_file ------
+

litellm_patch.py

@@ -0,0 +1,106 @@
+"""
+LiteLLM Async Task Cleanup Patch
+
+This module patches LiteLLM's asynchronous logging to ensure all tasks complete properly
+and prevent "Task was destroyed but it is pending!" errors.
+"""
+
+import asyncio
+import logging
+import functools
+import inspect
+import sys
+from typing import Any, Callable, Coroutine
+
+logger = logging.getLogger(__name__)
+
+# Global registry of pending LiteLLM async tasks
+_pending_tasks = set()
+
+def _patch_litellm_async_logging():
+    """
+    Patches LiteLLM async logging functions to ensure proper task cleanup.
+    Prevents "Task was destroyed but it is pending!" errors.
+    """
+    try:
+        # Try to import LiteLLM modules
+        import litellm
+        from litellm.utils import _client_async_logging_helper
+        
+        # Store original function
+        original_client_async_logging = _client_async_logging_helper
+        
+        # Create patched version with error handling
+        @functools.wraps(original_client_async_logging)
+        async def patched_client_async_logging_helper(*args, **kwargs):
+            try:
+                return await original_client_async_logging(*args, **kwargs)
+            except Exception as e:
+                logger.warning(f"LiteLLM async logging error (handled): {e}")
+                return None
+                
+        # Apply patch
+        litellm.utils._client_async_logging_helper = patched_client_async_logging_helper
+        
+        # Patch Logging class async_success_handler if available
+        if hasattr(litellm, 'litellm_core_utils') and hasattr(litellm.litellm_core_utils, 'litellm_logging'):
+            from litellm.litellm_core_utils.litellm_logging import Logging
+            
+            if hasattr(Logging, 'async_success_handler'):
+                original_async_success_handler = Logging.async_success_handler
+                
+                @functools.wraps(original_async_success_handler)
+                async def patched_async_success_handler(*args, **kwargs):
+                    try:
+                        return await original_async_success_handler(*args, **kwargs)
+                    except Exception as e:
+                        logger.warning(f"LiteLLM async_success_handler error (handled): {e}")
+                        return None
+                        
+                Logging.async_success_handler = patched_async_success_handler
+        
+        logger.info("Successfully patched LiteLLM async logging functions")
+        return True
+        
+    except ImportError:
+        logger.warning("Could not find LiteLLM modules to patch")
+        return False
+    except Exception as e:
+        logger.error(f"Error patching LiteLLM: {e}")
+        return False
+
+
+def create_task_with_cleanup(coro: Coroutine) -> asyncio.Task:
+    """
+    Creates an asyncio task with automatic cleanup registration.
+    Prevents orphaned tasks and associated warnings.
+    """
+    task = asyncio.create_task(coro)
+    _pending_tasks.add(task)
+    task.add_done_callback(_pending_tasks.discard)
+    return task
+
+
+async def cleanup_all_async_tasks(timeout: float = 2.0):
+    """
+    Waits for pending async tasks to complete within timeout period.
+    Should be called before exiting async contexts to prevent warnings.
+    """
+    if not _pending_tasks:
+        return
+    
+    logger.debug(f"Cleaning up {len(_pending_tasks)} pending async tasks...")
+    try:
+        # Wait for all pending tasks with a timeout
+        done, pending = await asyncio.wait(
+            _pending_tasks, timeout=timeout, return_when=asyncio.ALL_COMPLETED
+        )
+        
+        if pending:
+            logger.warning(f"{len(pending)} async tasks still pending after timeout")
+    except Exception as e:
+        logger.error(f"Error during async task cleanup: {e}")
+
+
+# Apply the patch when this module is imported
+_patch_litellm_async_logging()

my_config.py

@@ -0,0 +1,131 @@
+import os 
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+## Configuration
+
+class MyConfig:
+    pass
+
+MY_CONFIG = MyConfig ()
+
+## All of these settings can be overridden by .env file
+## And it will be loaded automatically by load_dotenv()
+## And they will take precedence over the default values below
+## See sample .env file 'env.sample.txt' for reference
+
+## HuggingFace config
+MY_CONFIG.HF_ENDPOINT = os.getenv("HF_ENDPOINT", "https://thealliance.ai/")
+
+## Crawl settings
+MY_CONFIG.WEBSITE_URL = os.getenv("WEBSITE_URL", "")
+MY_CONFIG.CRAWL_MAX_DOWNLOADS = int(os.getenv("CRAWL_MAX_DOWNLOADS", 100))
+MY_CONFIG.CRAWL_MAX_DEPTH = int(os.getenv("CRAWL_MAX_DEPTH", 3))
+MY_CONFIG.WAITTIME_BETWEEN_REQUESTS = float(os.getenv("WAITTIME_BETWEEN_REQUESTS", 0.1)) # in seconds
+MY_CONFIG.CRAWL_MIME_TYPE = 'text/html'
+
+
+## Directories
+MY_CONFIG.WORKSPACE_DIR = os.path.join(os.getenv('WORKSPACE_DIR', 'workspace'))
+MY_CONFIG.CRAWL_DIR = os.path.join( MY_CONFIG.WORKSPACE_DIR, "crawled")
+MY_CONFIG.PROCESSED_DATA_DIR = os.path.join( MY_CONFIG.WORKSPACE_DIR, "processed")
+
+## llama index will download the models to this directory
+os.environ["LLAMA_INDEX_CACHE_DIR"] = os.path.join(MY_CONFIG.WORKSPACE_DIR, "llama_index_cache")
+### -------------------------------
+
+# Find embedding models: https://huggingface.co/spaces/mteb/leaderboard
+
+MY_CONFIG.EMBEDDING_MODEL =  os.getenv("EMBEDDING_MODEL", 'ibm-granite/granite-embedding-30m-english')
+MY_CONFIG.EMBEDDING_LENGTH = int(os.getenv("EMBEDDING_LENGTH", 384))
+
+## Chunking
+MY_CONFIG.CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", 512))
+MY_CONFIG.CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", 20))
+
+
+### Milvus config  
+MY_CONFIG.COLLECTION_NAME = 'pages'
+
+# Separate Milvus databases for different RAG approaches
+# This allows running Vector RAG and Hybrid GraphRAG simultaneously without conflicts
+MY_CONFIG.MILVUS_URI_VECTOR = os.path.join( MY_CONFIG.WORKSPACE_DIR, 'vector_only_milvus.db')  # Vector RAG only
+MY_CONFIG.MILVUS_URI_HYBRID_GRAPH = os.path.join( MY_CONFIG.WORKSPACE_DIR, 'hybrid_graph_milvus.db')  # Hybrid GraphRAG
+
+# Vector Database Configuration
+MY_CONFIG.VECTOR_DB_TYPE = os.getenv("VECTOR_DB_TYPE", "cloud_zilliz")  # Options: "local" or "cloud_zilliz"
+
+# Zilliz Cloud Configuration (for cloud deployment)
+MY_CONFIG.ZILLIZ_CLUSTER_ENDPOINT = os.getenv("ZILLIZ_CLUSTER_ENDPOINT")
+MY_CONFIG.ZILLIZ_TOKEN = os.getenv("ZILLIZ_TOKEN")
+
+
+
+## ---- LLM settings ----
+## Choose one: We can do local or cloud LLMs
+# LLM_RUN_ENV controls which LLM backend to use: 'local_ollama' for local Ollama, 'cloud' for cloud LLMs
+# Set LLM_RUN_ENV in your .env file. Default is 'cloud' for production deployment.
+## Local LLMs are run on your machine using Ollama
+## Cloud LLMs are run on any LiteLLM supported service like Replicate / Nebius / Cerebras / etc
+## For running Ollama locally, please check the instructions in the docs/llm-local.md file
+
+
+MY_CONFIG.LLM_RUN_ENV = os.getenv("LLM_RUN_ENV", "cloud")
+
+
+MY_CONFIG.LLM_MODEL = os.getenv("LLM_MODEL", 'cerebras/llama3.1-8b')
+
+# Replicate API token (if using Replicate)
+MY_CONFIG.REPLICATE_API_TOKEN = os.getenv("REPLICATE_API_TOKEN", None)
+# Nebius API key (if using Nebius)
+MY_CONFIG.NEBIUS_API_KEY = os.getenv("NEBIUS_API_KEY", None)
+
+# --- GraphBuilder LLM API keys ---
+MY_CONFIG.CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY", None)
+MY_CONFIG.GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", None)
+
+# --- Graph entity/relationship extraction config ---
+MY_CONFIG.GRAPH_MIN_ENTITIES = int(os.getenv("GRAPH_MIN_ENTITIES", 5))
+MY_CONFIG.GRAPH_MAX_ENTITIES = int(os.getenv("GRAPH_MAX_ENTITIES", 15))
+MY_CONFIG.GRAPH_MIN_RELATIONSHIPS = int(os.getenv("GRAPH_MIN_RELATIONSHIPS", 3))
+MY_CONFIG.GRAPH_MAX_RELATIONSHIPS = int(os.getenv("GRAPH_MAX_RELATIONSHIPS", 8))
+MY_CONFIG.GRAPH_MIN_CONFIDENCE = float(os.getenv("GRAPH_MIN_CONFIDENCE", 0.8))
+MY_CONFIG.GRAPH_MAX_CONTENT_CHARS = int(os.getenv("GRAPH_MAX_CONTENT_CHARS", 12000))
+MY_CONFIG.GRAPH_SENTENCE_BOUNDARY_RATIO = float(os.getenv("GRAPH_SENTENCE_BOUNDARY_RATIO", 0.7))
+
+
+
+## --- GraphRAG ---
+# --- Neo4j config ---
+MY_CONFIG.NEO4J_URI = os.getenv("NEO4J_URI")
+MY_CONFIG.NEO4J_USER = os.getenv("NEO4J_USERNAME")
+MY_CONFIG.NEO4J_PASSWORD = os.getenv("NEO4J_PASSWORD")
+MY_CONFIG.NEO4J_DATABASE = os.getenv("NEO4J_DATABASE")
+MY_CONFIG.GRAPH_DATA_DIR = os.path.join(MY_CONFIG.WORKSPACE_DIR, "graph_data")
+
+
+
+
+
+## --- UI settings ---
+MY_CONFIG.STARTER_PROMPTS_STR = os.getenv("UI_STARTER_PROMPTS", 'What is this website?  |  What are upcoming events?  | Who are some of the partners?')
+
+
+MY_CONFIG.STARTER_PROMPTS = MY_CONFIG.STARTER_PROMPTS_STR.split("|") if MY_CONFIG.STARTER_PROMPTS_STR else []
+
+
+## --- Port Configuration ---
+# Flask apps (auto-configured via MY_CONFIG)
+MY_CONFIG.FLASK_VECTOR_PORT = int(os.getenv("FLASK_VECTOR_PORT", 8081))  # app_flask.py (vector RAG)
+MY_CONFIG.FLASK_GRAPH_PORT = int(os.getenv("FLASK_GRAPH_PORT", 8080))   # app_flask_graph.py (GraphRAG)
+
+# Chainlit apps (default port: 8000, custom ports for Docker deployments)
+MY_CONFIG.CHAINLIT_VECTOR_PORT = int(os.getenv("CHAINLIT_VECTOR_PORT", 8082))  # app_chainlit.py (Docker: 8082, Native: 8000)
+MY_CONFIG.CHAINLIT_GRAPH_PORT = int(os.getenv("CHAINLIT_GRAPH_PORT", 8083))    # app_chainlit_graph.py (Docker: 8083, Native: 8000)
+
+# Docker and external services
+MY_CONFIG.DOCKER_PORT = int(os.getenv("DOCKER_PORT", 8080))  # External host port (maps to DOCKER_APP_PORT)
+MY_CONFIG.DOCKER_APP_PORT = int(os.getenv("DOCKER_APP_PORT", 8080))  # Internal container port (all apps use this in Docker)
+MY_CONFIG.OLLAMA_PORT = int(os.getenv("OLLAMA_PORT", 11434))  # Ollama server port

news.md

@@ -0,0 +1,51 @@
+# Allycat News
+
+## 2025-10-17: GraphRAG
+
+Niloy Deb Barma from [Vel Tech University](https://www.veltech.edu.in/btech-admissions/) in Chennai is the process of adding GraphRAG to AllyCat.  
+
+## 2025-08-23: Expanded Team
+
+In addition to Sujee and Dave, we have begun working with Chirag, Nikhil and the team at [Open Governance](https://www.opgov.ai)
+
+## 2025-07-14: Big Update 
+
+Lot of cool updates:
+
+**Robust web crawler** ([#31](https://github.com/The-AI-Alliance/allycat/issues/31))
+
+Completely redid web crawler.  Now it 
+- is more robust and handle scenarios that made previous crawler fail.
+- can handle multiple file types (not just text/html) correctly
+- Handle anchor tags (`a.html#news`) in HTML files correctly 
+- pauses (customizable) between requests so to not hammer the webserver.
+  
+**using [LiteLLM](https://docs.litellm.ai/docs/) for LLM inference** ([#34](https://github.com/The-AI-Alliance/allycat/issues/34))
+
+This allows us to seamlessly access LLMs running locally (using [ollama](https://ollama.com/)) or calling inference providers like Nebius, Replicate ..etc.
+
+Also singinficantly simplified LLM configuration.
+
+**Expanded support for many file types (pdf, docx)** ([#37](https://github.com/The-AI-Alliance/allycat/issues/37))
+
+Before we just handled HTML files. Now we can download and process other popular file types - like PDF, DOCX ..etc.  We use [Docling](https://github.com/docling-project/docling) for processing files.
+
+
+**Added [uv](https://docs.astral.sh/uv/) package manager support** ([#26](https://github.com/The-AI-Alliance/allycat/issues/26))
+
+UV will be the preferred package manager going forward.  We will still maintain `requirements.txt` to support other package managers.
+
+
+**Better config management**([#19](https://github.com/The-AI-Alliance/allycat/issues/19))
+
+Lot of user configuration can be set using `.env` file.  This simplifies config management and allows for easier and faster experimentation without changing code.
+
+
+**Documentation update**
+
+Various doc updates.
+
+**Huge thanks to all the contributors**
+
+- [Steven Pousty](https://github.com/thesteve0)  ([linkedin](https://www.linkedin.com/in/thesteve0/))
+- [Santosh Borse](https://github.com/santoshborse)

package-lock.json

@@ -0,0 +1,523 @@
+{
+  "name": "allycat",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "dotenv": "^17.2.1",
+        "neo4j": "^2.0.0-RC2"
+      }
+    },
+    "node_modules/ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "license": "MIT",
+      "dependencies": {
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
+      }
+    },
+    "node_modules/asn1": {
+      "version": "0.2.6",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
+      "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": "~2.1.0"
+      }
+    },
+    "node_modules/assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "license": "MIT"
+    },
+    "node_modules/aws-sign2": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
+      "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/aws4": {
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz",
+      "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==",
+      "license": "MIT"
+    },
+    "node_modules/bcrypt-pbkdf": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
+      "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "tweetnacl": "^0.14.3"
+      }
+    },
+    "node_modules/caseless": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
+      "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "license": "MIT",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==",
+      "license": "MIT"
+    },
+    "node_modules/dashdash": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+      "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==",
+      "license": "MIT",
+      "dependencies": {
+        "assert-plus": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/dotenv": {
+      "version": "17.2.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.1.tgz",
+      "integrity": "sha512-kQhDYKZecqnM0fCnzI5eIv5L4cAe/iRI+HqMbO/hbRdTAeXDG+M9FjipUxNfbARuEg4iHIbhnhs78BCHNbSxEQ==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
+    "node_modules/ecc-jsbn": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
+      "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==",
+      "license": "MIT",
+      "dependencies": {
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.1.0"
+      }
+    },
+    "node_modules/extend": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
+      "license": "MIT"
+    },
+    "node_modules/extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==",
+      "engines": [
+        "node >=0.6.0"
+      ],
+      "license": "MIT"
+    },
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "license": "MIT"
+    },
+    "node_modules/fast-json-stable-stringify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "license": "MIT"
+    },
+    "node_modules/forever-agent": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
+      "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/form-data": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.6",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 0.12"
+      }
+    },
+    "node_modules/getpass": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+      "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==",
+      "license": "MIT",
+      "dependencies": {
+        "assert-plus": "^1.0.0"
+      }
+    },
+    "node_modules/har-schema": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
+      "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/har-validator": {
+      "version": "5.1.5",
+      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
+      "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
+      "deprecated": "this library is no longer supported",
+      "license": "MIT",
+      "dependencies": {
+        "ajv": "^6.12.3",
+        "har-schema": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/http-signature": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
+      "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==",
+      "license": "MIT",
+      "dependencies": {
+        "assert-plus": "^1.0.0",
+        "jsprim": "^1.2.2",
+        "sshpk": "^1.7.0"
+      },
+      "engines": {
+        "node": ">=0.8",
+        "npm": ">=1.3.7"
+      }
+    },
+    "node_modules/is-typedarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
+      "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==",
+      "license": "MIT"
+    },
+    "node_modules/isstream": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+      "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==",
+      "license": "MIT"
+    },
+    "node_modules/jsbn": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+      "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==",
+      "license": "MIT"
+    },
+    "node_modules/json-schema": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz",
+      "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==",
+      "license": "(AFL-2.1 OR BSD-3-Clause)"
+    },
+    "node_modules/json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "license": "MIT"
+    },
+    "node_modules/json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
+      "license": "ISC"
+    },
+    "node_modules/jsprim": {
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz",
+      "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==",
+      "license": "MIT",
+      "dependencies": {
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.4.0",
+        "verror": "1.10.0"
+      },
+      "engines": {
+        "node": ">=0.6.0"
+      }
+    },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/neo4j": {
+      "version": "2.0.0-RC2",
+      "resolved": "https://registry.npmjs.org/neo4j/-/neo4j-2.0.0-RC2.tgz",
+      "integrity": "sha512-TTTRwv8t3S0Mp6rVtgY4RNt+SCSl+ccuXhP6DmXERtNp5Vs8LlJ85uZiGJKcT74Xthqc4ihl517+bBOqQJxhNA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "request": "^2.27.0",
+        "underscore": "1.7.x"
+      },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
+    "node_modules/oauth-sign": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
+      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/performance-now": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
+      "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==",
+      "license": "MIT"
+    },
+    "node_modules/psl": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz",
+      "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==",
+      "license": "MIT",
+      "dependencies": {
+        "punycode": "^2.3.1"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/lupomontero"
+      }
+    },
+    "node_modules/punycode": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/qs": {
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz",
+      "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==",
+      "license": "BSD-3-Clause",
+      "engines": {
+        "node": ">=0.6"
+      }
+    },
+    "node_modules/request": {
+      "version": "2.88.2",
+      "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
+      "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
+      "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "aws-sign2": "~0.7.0",
+        "aws4": "^1.8.0",
+        "caseless": "~0.12.0",
+        "combined-stream": "~1.0.6",
+        "extend": "~3.0.2",
+        "forever-agent": "~0.6.1",
+        "form-data": "~2.3.2",
+        "har-validator": "~5.1.3",
+        "http-signature": "~1.2.0",
+        "is-typedarray": "~1.0.0",
+        "isstream": "~0.1.2",
+        "json-stringify-safe": "~5.0.1",
+        "mime-types": "~2.1.19",
+        "oauth-sign": "~0.9.0",
+        "performance-now": "^2.1.0",
+        "qs": "~6.5.2",
+        "safe-buffer": "^5.1.2",
+        "tough-cookie": "~2.5.0",
+        "tunnel-agent": "^0.6.0",
+        "uuid": "^3.3.2"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT"
+    },
+    "node_modules/safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+      "license": "MIT"
+    },
+    "node_modules/sshpk": {
+      "version": "1.18.0",
+      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
+      "integrity": "sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==",
+      "license": "MIT",
+      "dependencies": {
+        "asn1": "~0.2.3",
+        "assert-plus": "^1.0.0",
+        "bcrypt-pbkdf": "^1.0.0",
+        "dashdash": "^1.12.0",
+        "ecc-jsbn": "~0.1.1",
+        "getpass": "^0.1.1",
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.0.2",
+        "tweetnacl": "~0.14.0"
+      },
+      "bin": {
+        "sshpk-conv": "bin/sshpk-conv",
+        "sshpk-sign": "bin/sshpk-sign",
+        "sshpk-verify": "bin/sshpk-verify"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/tough-cookie": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
+      "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "psl": "^1.1.28",
+        "punycode": "^2.1.1"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/tunnel-agent": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+      "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "safe-buffer": "^5.0.1"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/tweetnacl": {
+      "version": "0.14.5",
+      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
+      "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==",
+      "license": "Unlicense"
+    },
+    "node_modules/underscore": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz",
+      "integrity": "sha512-cp0oQQyZhUM1kpJDLdGO1jPZHgS/MpzoWYfe9+CM2h/QGDZlqwT2T3YGukuBdaNJ/CAPoeyAZRRHz8JFo176vA=="
+    },
+    "node_modules/uri-js": {
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+      "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "punycode": "^2.1.0"
+      }
+    },
+    "node_modules/uuid": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
+      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "deprecated": "Please upgrade  to version 7 or higher.  Older versions may use Math.random() in certain circumstances, which is known to be problematic.  See https://v8.dev/blog/math-random for details.",
+      "license": "MIT",
+      "bin": {
+        "uuid": "bin/uuid"
+      }
+    },
+    "node_modules/verror": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==",
+      "engines": [
+        "node >=0.6.0"
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "assert-plus": "^1.0.0",
+        "core-util-is": "1.0.2",
+        "extsprintf": "^1.2.0"
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,6 @@
+{
+  "dependencies": {
+    "dotenv": "^17.2.1",
+    "neo4j": "^2.0.0-RC2"
+  }
+}

pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "allycat-1"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "chainlit>=2.2.1",
+    "docling>=2.41.0",
+    "flask>=3.1.1",
+    "humanfriendly>=10.0",
+    "litellm>=1.74.3",
+    "llama-index>=0.12.48",
+    "llama-index-embeddings-huggingface>=0.5.5",
+    "llama-index-llms-litellm>=0.5.1",
+    "llama-index-vector-stores-milvus>=0.8.5",
+    "milvus-lite>=2.5.1",
+    "mimetypes-magic>=0.4.30",
+    "nest-asyncio>=1.6.0",
+    "pandas>=2.3.1",
+    "pymilvus>=2.5.12",
+    "tqdm>=4.67.1",
+]
+
+[dependency-groups]
+dev = [
+    "ipykernel>=6.29.5",
+]

query_utils.py

@@ -0,0 +1,12 @@
+def tweak_query(query : str, model : str):
+    """
+    for qwen3 models, turn off thinking
+    """
+    
+    # Check if the model is qwen3
+    if 'qwen3' in model:
+        # Check if the query contains '/no_think'
+        if '/no_think' not in query:
+            # Append '/no_think' to the query
+            query += '\n/no_think'
+    return query

requirements-build.txt

@@ -0,0 +1,35 @@
+# ============================================
+# BUILD/PIPELINE REQUIREMENTS - Full Pipeline
+# ============================================
+# These packages are ONLY needed for running the pipeline:
+# - Crawling websites (1_crawl_site.py)
+# - Processing files (2_process_files.py) 
+# - Graph extraction (2b_process_graph_phase*.py)
+# - Saving to databases (3*.py)
+#
+# After pipeline completes, these can be REMOVED to save ~300-500 MB RAM
+
+# ============================================
+# Document Processing - REMOVE AFTER PIPELINE
+# ============================================
+docling                 # PDF/HTML to markdown conversion (~100 MB)
+html2text              # HTML processing (~10 MB)
+
+# ============================================
+# Graph Community Detection - REMOVE AFTER PIPELINE
+# ============================================
+igraph                 # Graph analysis library (~50 MB)
+leidenalg             # Leiden algorithm for communities (~30 MB)
+graspologic           # Graph statistics (~40 MB)
+
+# ============================================
+# Development Tools - REMOVE AFTER PIPELINE
+# ============================================
+milvus-lite==2.4.11    # Local Milvus server (not needed if using cloud Zilliz) (~100 MB)
+tqdm                   # Progress bars (nice to have, but not essential)
+ipykernel             # Jupyter support (only for development)
+fastmcp               # MCP support (only for development)
+
+# ============================================
+# Total Savings if Removed: ~350-500 MB
+# ============================================

requirements-docker-cloud.txt

@@ -0,0 +1,49 @@
+# Core document processing (lightweight)
+docling
+html2text
+
+# Asyncio support
+nest_asyncio
+
+# PyTorch CPU-only (much smaller than GPU)
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.6.0+cpu
+
+# JSON parsing
+orjson>=3.8.0
+json-repair>=0.7.0
+
+# Vector DB - Cloud Zilliz support
+pymilvus==2.5.5
+
+# LLM Integration
+litellm
+
+# LlamaIndex
+llama-index
+llama-index-embeddings-huggingface
+llama-index-llms-litellm
+llama-index-vector-stores-milvus==0.5.0
+
+# Graph Database
+neo4j
+networkx
+python-louvain
+igraph
+leidenalg
+graspologic
+
+# Graph LLM APIs
+google-generativeai
+openai>=1.0.0
+fastmcp
+
+# Web Framework
+flask==2.3.3
+chainlit
+
+# Utilities
+python-dotenv
+humanfriendly
+pandas
+tqdm

requirements-docker.txt

@@ -0,0 +1,51 @@
+# Core document processing
+docling
+html2text
+
+# Asyncio support
+nest_asyncio
+
+# PyTorch CPU-only (much smaller than GPU)
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.6.0+cpu
+
+# JSON parsing
+orjson>=3.8.0
+json-repair>=0.7.0
+
+# Vector DB - Supports both cloud (Zilliz) and local (Milvus Lite)
+pymilvus==2.5.5
+milvus-lite==2.4.11
+
+# LLM Integration
+litellm
+
+# LlamaIndex
+llama-index
+llama-index-embeddings-huggingface
+llama-index-llms-litellm
+llama-index-llms-ollama
+llama-index-vector-stores-milvus==0.5.0
+
+# Graph Database
+neo4j
+networkx
+python-louvain
+igraph
+leidenalg
+graspologic
+
+# Graph LLM APIs
+google-generativeai
+openai>=1.0.0
+fastmcp
+
+# Web Frameworks
+flask==2.3.3
+chainlit
+
+# Utilities
+python-dotenv
+humanfriendly
+pandas
+tqdm

requirements-runtime.txt

@@ -0,0 +1,59 @@
+# ============================================
+# RUNTIME REQUIREMENTS - Flask GraphRAG App
+# ============================================
+# These are the MINIMAL packages needed to run the Flask GraphRAG app
+# after the pipeline has been completed.
+# Use this for production deployments to save ~500 MB RAM
+
+# ============================================
+# Core Runtime - DO NOT REMOVE
+# ============================================
+
+# Asyncio support
+nest_asyncio
+
+# Advanced JSON parsing (for LLM responses)
+orjson>=3.8.0
+json-repair>=0.7.0
+
+# Vector Database (client-only, lightweight)
+pymilvus==2.5.5
+
+# LLM Integration
+litellm
+
+# LlamaIndex Core (for querying)
+llama-index
+llama-index-embeddings-huggingface
+llama-index-llms-litellm
+llama-index-llms-ollama
+llama-index-vector-stores-milvus==0.5.0
+
+# Graph Database (client-only)
+neo4j
+
+# Graph Analysis (for community queries)
+networkx
+
+# LLM APIs (for query synthesis)
+google-generativeai
+openai>=1.0.0
+
+# Web Framework
+flask==2.3.3
+chainlit              # Chat UI (only if using chainlit apps)
+
+# Utilities
+python-dotenv
+humanfriendly
+pandas
+
+# ============================================
+# Embedding Model - LARGE (500+ MB)
+# ============================================
+# PyTorch CPU (required for sentence-transformers/embeddings)
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.6.0+cpu
+
+# Note: For cloud embeddings in future, remove torch + sentence-transformers
+# to save ~500 MB

requirements.txt

@@ -0,0 +1,51 @@
+# Core document processing
+docling
+html2text
+# Asyncio support
+nest_asyncio
+
+# PyTorch CPU-only (much smaller than GPU)
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.6.0+cpu
+
+# Advanced JSON parsing
+orjson>=3.8.0
+json-repair>=0.7.0
+
+# Vector Database
+pymilvus==2.5.5
+milvus-lite==2.4.11
+
+# LLM Integration
+litellm
+
+# LlamaIndex
+llama-index
+llama-index-embeddings-huggingface
+llama-index-llms-litellm
+llama-index-llms-ollama
+llama-index-vector-stores-milvus==0.5.0
+
+# Graph Database
+neo4j
+networkx
+python-louvain
+igraph
+leidenalg
+graspologic
+
+# Graph LLM APIs
+google-generativeai
+openai>=1.0.0
+fastmcp
+
+# Web Frameworks
+flask==2.3.3
+chainlit
+
+# Utilities
+python-dotenv
+humanfriendly
+pandas
+tqdm
+ipykernel