feat: implement tiktoken env variables

.gitignore

@@ -66,6 +66,7 @@ LightRAG_2-4.pdf
 download_models_hf.py
 lightrag-dev/
 gui/
+tiktoken_cache/
 
 # unit-test files
 test_*

env.example

@@ -10,6 +10,12 @@ OLLAMA_EMULATING_MODEL_TAG=latest
 # WORKERS=2
 # CORS_ORIGINS=http://localhost:3000,http://localhost:8080
 
+### Tiktoken Cache Configuration (for offline deployment)
+### Set this to a local directory containing cached tiktoken models
+### This prevents tiktoken from downloading models from the internet on initialization
+### See docs/offline_setup.md for setup instructions
+# TIKTOKEN_CACHE_DIR=./tiktoken_cache
+
 ### Login Configuration
 # AUTH_ACCOUNTS='admin:admin123,user1:pass456'
 # TOKEN_SECRET=Your-Key-For-LightRAG-API-Server

raganything/raganything.py

@@ -14,18 +14,19 @@ import asyncio
 import atexit
 from dataclasses import dataclass, field
 from pathlib import Path
+from dotenv import load_dotenv
 
 # Add project root directory to Python path
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
-from lightrag import LightRAG
-from lightrag.utils import logger
-from dotenv import load_dotenv
-
-# Load environment variables from .env file
+# Load environment variables from .env file BEFORE importing LightRAG
+# This is critical for TIKTOKEN_CACHE_DIR to work properly in offline environments
 # The OS environment variables take precedence over the .env file
 load_dotenv(dotenv_path=".env", override=False)
 
+from lightrag import LightRAG
+from lightrag.utils import logger
+
 # Import configuration and modules
 from raganything.config import RAGAnythingConfig
 from raganything.query import QueryMixin