feat(python): add recursive option to from_dir method

zTgx · zTgx · commit d763949edfb3 · 2026-04-15T10:05:14.000+08:00
Add recursive parameter to PyIndexContext.from_dir with default value false.
When recursive=True, uses IndexContext::from_dir_recursive instead of
from_dir.

feat(rust): implement recursive directory scanning for IndexContext

Add IndexContext::from_dir_recursive method that scans directories
recursively. Refactor from_dir to use internal scan_dir helper function.
Update supported extensions to only include .md and .pdf files.

docs: add directory indexing example with CLI interface

Add new example showing how to recursively index documents in a directory
with command line arguments for recursive/non-recursive modes.
diff --git a/python/src/lib.rs b/python/src/lib.rs
@@ -219,11 +219,19 @@ impl PyIndexContext {
     }
 
     /// Create an IndexContext from all supported files in a directory.
+    ///
+    /// Args:
+    ///     path: Directory path to scan.
+    ///     recursive: If True, scan subdirectories recursively. Default: False.
     #[staticmethod]
-    fn from_dir(path: String) -> Self {
-        Self {
-            inner: IndexContext::from_dir(&path),
-        }
+    #[pyo3(signature = (path, recursive=false))]
+    fn from_dir(path: String, recursive: bool) -> Self {
+        let inner = if recursive {
+            IndexContext::from_dir_recursive(&path)
+        } else {
+            IndexContext::from_dir(&path)
+        };
+        Self { inner }
     }
 
     /// Create an IndexContext from text content.
diff --git a/rust/examples/index_directory.rs b/rust/examples/index_directory.rs
@@ -0,0 +1,123 @@
+// Copyright (c) 2026 vectorless developers
+// SPDX-License-Identifier: Apache-2.0
+
+//! Directory indexing example — recursively index all documents in a directory.
+//!
+//! ```bash
+//! # Using environment variables for LLM config:
+//! LLM_API_KEY=sk-xxx LLM_MODEL=google/gemini-3-flash-preview \
+//!   LLM_ENDPOINT=http://localhost:4000/api/v1 \
+//!   cargo run --example index_directory -- /path/to/docs
+//!
+//! # With recursive flag (default):
+//! cargo run --example index_directory -- /path/to/docs --recursive
+//!
+//! # Non-recursive (top-level only):
+//! cargo run --example index_directory -- /path/to/docs --no-recursive
+//! ```
+
+use vectorless::{EngineBuilder, IndexContext};
+
+#[tokio::main]
+async fn main() -> vectorless::Result<()> {
+    tracing_subscriber::fmt::init();
+
+    // Parse CLI arguments
+    let args: Vec<String> = std::env::args().collect();
+    let dir = args
+        .get(1)
+        .map(|s| s.as_str())
+        .unwrap_or("./samples");
+    let recursive = !args.iter().any(|a| a == "--no-recursive");
+
+    // Build engine
+    let api_key = std::env::var("LLM_API_KEY").unwrap_or_else(|_| "sk-or-v1-...".to_string());
+    let model =
+        std::env::var("LLM_MODEL").unwrap_or_else(|_| "google/gemini-3-flash-preview".to_string());
+    let endpoint = std::env::var("LLM_ENDPOINT")
+        .unwrap_or_else(|_| "http://localhost:4000/api/v1".to_string());
+
+    let engine = EngineBuilder::new()
+        .with_workspace("./workspace_directory_example")
+        .with_key(&api_key)
+        .with_model(&model)
+        .with_endpoint(&endpoint)
+        .build()
+        .await
+        .map_err(|e| vectorless::Error::Config(e.to_string()))?;
+
+    // Index directory
+    let ctx = if recursive {
+        println!("Recursively indexing: {}", dir);
+        IndexContext::from_dir_recursive(dir)
+    } else {
+        println!("Indexing top-level files in: {}", dir);
+        IndexContext::from_dir(dir)
+    };
+
+    if ctx.is_empty() {
+        println!("No supported files found in: {}", dir);
+        return Ok(());
+    }
+
+    println!("Found {} file(s) to index", ctx.len());
+
+    let result = engine.index(ctx).await?;
+
+    println!("\nIndexed {} document(s):", result.items.len());
+    for item in &result.items {
+        println!("  {} ({})", item.name, item.doc_id);
+        if let Some(metrics) = &item.metrics {
+            println!(
+                "    nodes: {}, time: {}ms",
+                metrics.nodes_processed,
+                metrics.total_time_ms()
+            );
+        }
+    }
+
+    if result.has_failures() {
+        println!("\nFailed:");
+        for f in &result.failed {
+            println!("  {} — {}", f.source, f.error);
+        }
+    }
+
+    // Query across all indexed documents
+    let query = "What is this about?";
+    println!("\nQuerying: \"{query}\"");
+
+    let answer = engine
+        .query(vectorless::QueryContext::new(query))
+        .await?;
+
+    for item in &answer.items {
+        println!("  [{} score={:.2}]", item.doc_id, item.score);
+        let preview: String = item.content.chars().take(200).collect();
+        println!("  {preview}");
+        if item.content.len() > 200 {
+            println!("  ...");
+        }
+    }
+
+    // Metrics report
+    let report = engine.metrics_report();
+    println!("\nMetrics:");
+    println!(
+        "  LLM: {} calls, {} tokens, ${:.4}",
+        report.llm.total_calls,
+        report.llm.total_tokens,
+        report.llm.estimated_cost_usd,
+    );
+    println!(
+        "  Retrieval: {} queries, avg score {:.2}",
+        report.retrieval.total_queries, report.retrieval.avg_path_score,
+    );
+
+    // Cleanup
+    for doc in engine.list().await? {
+        engine.remove(&doc.id).await?;
+    }
+
+    Ok(())
+}
diff --git a/rust/src/client/index_context.rs b/rust/src/client/index_context.rs
@@ -29,7 +29,11 @@
 //! ```rust,no_run
 //! use vectorless::client::IndexContext;
 //!
+//! // Non-recursive (top-level only)
 //! let ctx = IndexContext::from_dir("./documents");
+//!
+//! // Recursive (includes subdirectories)
+//! let ctx = IndexContext::from_dir_recursive("./documents");
 //! ```
 
 use std::path::PathBuf;
@@ -149,27 +153,58 @@ impl IndexContext {
     /// Create from a directory path.
     ///
     /// Indexes all supported files in the directory (non-recursive).
-    /// Supported extensions: `.md`, `.pdf`, `.txt`.
+    /// Supported extensions: `.md`, `.pdf`.
     pub fn from_dir(dir: impl Into<PathBuf>) -> Self {
+        Self::scan_dir(dir, false)
+    }
+
+    /// Create from a directory path with recursive scanning.
+    ///
+    /// Recursively indexes all supported files in the directory and its
+    /// subdirectories. Supported extensions: `.md`, `.pdf`.
+    pub fn from_dir_recursive(dir: impl Into<PathBuf>) -> Self {
+        Self::scan_dir(dir, true)
+    }
+
+    /// Internal: scan a directory for supported document files.
+    fn scan_dir(dir: impl Into<PathBuf>, recursive: bool) -> Self {
         let dir = dir.into();
-        let supported_extensions = ["md", "markdown", "pdf", "txt"];
+        let supported_extensions = ["md", "pdf"];
 
         let mut sources = Vec::new();
-        if let Ok(entries) = std::fs::read_dir(&dir) {
+        Self::collect_files(&dir, &supported_extensions, recursive, &mut sources);
+
+        Self {
+            sources,
+            name: None,
+            options: IndexOptions::default(),
+        }
+    }
+
+    /// Recursively or non-recursively collect supported files.
+    fn collect_files(
+        dir: &std::path::Path,
+        extensions: &[&str],
+        recursive: bool,
+        sources: &mut Vec<IndexSource>,
+    ) {
+        if let Ok(entries) = std::fs::read_dir(dir) {
+            let mut subdirs = Vec::new();
             for entry in entries.flatten() {
                 let path = entry.path();
-                if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
-                    if supported_extensions.contains(&ext.to_lowercase().as_str()) {
+                if path.is_dir() {
+                    if recursive {
+                        subdirs.push(path);
+                    }
+                } else if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
+                    if extensions.contains(&ext.to_lowercase().as_str()) {
                         sources.push(IndexSource::Path(path));
                     }
                 }
             }
-        }
-
-        Self {
-            sources,
-            name: None,
-            options: IndexOptions::default(),
+            for subdir in subdirs {
+                Self::collect_files(&subdir, extensions, recursive, sources);
+            }
         }
     }
 
@@ -316,4 +351,32 @@ mod tests {
         let ctx = IndexContext::from(PathBuf::from("./test.md"));
         assert_eq!(ctx.len(), 1);
     }
+
+    #[test]
+    fn test_from_dir_recursive() {
+        // Create a temp directory structure:
+        //   tmp/
+        //     a.md
+        //     sub/
+        //       b.md
+        //       deep/
+        //         c.pdf
+        let tmp = std::env::temp_dir().join("vectorless_test_dir_recursive");
+        let _ = std::fs::remove_dir_all(&tmp);
+        std::fs::create_dir_all(tmp.join("sub/deep")).unwrap();
+        std::fs::write(tmp.join("a.md"), "# A").unwrap();
+        std::fs::write(tmp.join("sub/b.md"), "# B").unwrap();
+        std::fs::write(tmp.join("sub/deep/c.pdf"), b"%PDF").unwrap();
+        std::fs::write(tmp.join("sub/deep/ignore.dat"), b"xxx").unwrap();
+
+        // Non-recursive: only top-level
+        let ctx = IndexContext::from_dir(&tmp);
+        assert_eq!(ctx.len(), 1); // only a.md
+
+        // Recursive: all levels
+        let ctx = IndexContext::from_dir_recursive(&tmp);
+        assert_eq!(ctx.len(), 3); // a.md, b.md, c.pdf
+
+        let _ = std::fs::remove_dir_all(&tmp);
+    }
 }
