Improve symbol_insights and path_search

Better instructions and more clear functionality and better tool schema

Author: DolceTriade

src/mcp/server.rs

@@ -90,7 +90,7 @@ fn mcp_docs_payload() -> Value {
             "or_support": "Use any_terms:[...] for OR semantics. all_terms are ANDed.",
             "wildcards": "Use path/file as glob-like filters. all_terms and any_terms are literal terms, not wildcard patterns.",
             "regex": "Use the regex field for content regex matching. Provide only the pattern string and JSON-escape backslashes.",
-            "path_search_behavior": "path_search requires a non-empty plain fuzzy query and is for fuzzy path matching only.",
+            "path_search_behavior": "path_search requires a non-empty plain query string and performs case-insensitive substring matching on paths only.",
             "file_list_behavior": "file_list enumerates directories and files with optional recursive depth and limit. Use path as a directory prefix, not a search query.",
             "file_content_behavior": "file_content supports optional start_line/end_line (1-based, inclusive) to return snippets instead of full files.",
             "recency_workflow": "For recent or older change questions: repositories -> repo_branches -> search by branch and compare indexed_at or is_live.",
@@ -116,7 +116,7 @@ fn mcp_docs_payload() -> Value {
                 "No branch results: call repo_branches and use the exact branch name.",
                 "Need OR behavior: place alternatives in any_terms:[\"termA\",\"termB\"].",
                 "Need regex matching: set the regex field instead of using wildcard plain terms.",
-                "Need directory listing: use file_list. Need fuzzy path lookup: use path_search."
+                "Need directory listing: use file_list. Need case-insensitive substring path lookup: use path_search."
             ]
         },
         "cookbook": [
@@ -126,7 +126,7 @@ fn mcp_docs_payload() -> Value {
             "4) search({repo, branch, historical:true, all_terms:[\"term\"]}) for older snapshots",
             "5) search({repo, regex:\"\\\\bQueryParser\\\\(\"}) for regex matching",
             "6) file_list({repo, branch, path:\"src/mcp\", depth:2}) for directory enumeration",
-            "7) path_search({repo, branch, query:\"mcp serv\"}) for fuzzy path lookup",
+            "7) path_search({repo, branch, query:\"mcp_serv\"}) for case-insensitive substring path lookup",
             "8) file_content({repo, branch, path, start_line?, end_line?}) for raw source text or snippets",
             "9) For large files, prefer file_content with line snippets first, then expand only if needed",
             "10) symbol_insights({params:{...}}) for definitions and references",
@@ -199,7 +199,7 @@ async fn mcp_rpc(Json(req): Json<JsonRpcRequest>) -> Response {
                     "name": "pointer-mcp",
                     "version": env!("CARGO_PKG_VERSION"),
                 },
-                "instructions": "Use tools to query indexed code and symbol information; do not fall back to local filesystem reads for indexed lookup. Operational flow: repositories -> repo_branches -> file_list/path_search -> file_content/search/symbol_insights. search accepts structured JSON fields only; do not send a free-form `query` string. Keep filter values plain: do not include prefixes like `repo:`, `path:`, or `regex:` inside field values. all_terms are AND semantics and any_terms are OR semantics (fanout + dedupe). For recency/version questions like 'recent change', call repo_branches first, then run search with explicit branch values and compare indexed_at/is_live metadata; add historical:true when historical snapshots should be included. Plain terms do not support wildcard matching; use the regex field for pattern matching. path_search requires a non-empty plain fuzzy query and is not a directory listing endpoint; use file_list for enumeration. For large files, call file_content with start_line/end_line first to limit context size.",
+                "instructions": "Use tools to query indexed code and symbol information; do not fall back to local filesystem reads for indexed lookup. Operational flow: repositories -> repo_branches -> file_list/path_search -> file_content/search/symbol_insights. search accepts structured JSON fields only; do not send a free-form `query` string. Keep filter values plain: do not include prefixes like `repo:`, `path:`, or `regex:` inside field values. all_terms are AND semantics and any_terms are OR semantics (fanout + dedupe). For recency/version questions like 'recent change', call repo_branches first, then run search with explicit branch values and compare indexed_at/is_live metadata; add historical:true when historical snapshots should be included. Plain terms do not support wildcard matching; use the regex field for pattern matching. path_search requires a non-empty plain query string and performs case-insensitive substring matching over paths; it is not a directory listing endpoint, so use file_list for enumeration. For large files, call file_content with start_line/end_line first to limit context size.",
             });
             jsonrpc_result(req.id, result)
         }
@@ -338,7 +338,7 @@ fn mcp_tools() -> Vec<Value> {
                     "repo": { "type": "string", "description": "Exact repository key from repositories. Example: \"pointer\"." },
                     "branch": { "type": "string", "description": "Exact branch name from repo_branches. Example: \"main\"." },
                     "lang": { "type": "string", "description": "Language filter. Example: \"rust\"." },
-                    "path": { "type": "string", "description": "Glob-like path filter only. Example: \"src/mcp/**\". Do not use this for fuzzy lookup." },
+                    "path": { "type": "string", "description": "Glob-like path filter only. Example: \"src/mcp/**\". Do not use this for path substring lookup." },
                     "file": { "type": "string", "description": "Glob-like filename/path filter. Example: \"*.rs\"." },
                     "regex": { "type": "string", "description": "Content regex pattern only. Do not prefix with `regex:`. JSON-escape backslashes, for example \"\\\\bQueryParser\\\\(\"." },
                     "case": { "type": "string", "enum": ["yes", "no", "auto"] },
@@ -404,36 +404,48 @@ fn mcp_tools() -> Vec<Value> {
         }),
         json!({
             "name": "file_list",
-            "description": "Enumerate files/directories under a path for a repository+branch from the index. Supports bounded recursive traversal with depth and limit. Use this for directory listing workflows and then call file_content on specific files. `path` is a directory prefix, not a fuzzy search query. Response includes truncated flag, branch freshness, and stable paths.",
+            "description": "Enumerate files/directories under a path for a repository+branch from the index. Supports bounded recursive traversal with depth and limit. Use this for directory listing workflows and then call file_content on specific files. `path` is a directory prefix, not a substring search query. Optional cursor/auto_paginate can fetch more results while preserving truncation and narrowing guidance. Response includes truncated flag, branch freshness, and stable paths.",
             "inputSchema": {
                 "type": "object",
                 "properties": {
                     "repo": { "type": "string" },
                     "branch": { "type": "string" },
-                    "path": { "type": "string", "description": "Directory prefix to enumerate from. Example: \"src/mcp\". Do not send a fuzzy query here." },
+                    "path": { "type": "string", "description": "Directory prefix to enumerate from. Example: \"src/mcp\". Do not send a substring search query here." },
                     "depth": { "type": "integer", "minimum": 1, "maximum": 10 },
-                    "limit": { "type": "integer", "minimum": 1, "maximum": 5000 }
+                    "limit": { "type": "integer", "minimum": 1, "maximum": 5000 },
+                    "cursor": { "type": "string", "description": "Optional pagination cursor from prior file_list response." },
+                    "auto_paginate": { "type": "boolean", "description": "If true, server attempts to return multiple pages up to max_pages/max_total_entries." },
+                    "max_pages": { "type": "integer", "minimum": 1, "maximum": 5 },
+                    "max_total_entries": { "type": "integer", "minimum": 1, "maximum": 5000 }
                 },
                 "examples": [
-                    { "repo": "pointer", "branch": "main", "path": "src/mcp", "depth": 2, "limit": 100 }
+                    { "repo": "pointer", "branch": "main", "path": "src/mcp", "depth": 2, "limit": 100 },
+                    { "repo": "pointer", "branch": "main", "path": "src", "limit": 100, "cursor": "100" },
+                    { "repo": "pointer", "branch": "main", "path": "src", "limit": 100, "auto_paginate": true, "max_pages": 3, "max_total_entries": 250 }
                 ],
                 "required": ["repo", "branch"],
                 "additionalProperties": false
             }
         }),
         json!({
             "name": "path_search",
-            "description": "Search file and directory paths within a repository and branch using a non-empty plain fuzzy query (fuzzy path lookup). This is path-only matching and does not enumerate full directory contents; use file_list for enumeration and file_content for file bodies. Do not send filter syntax like `path:` or glob patterns here. Includes freshness metadata.",
+            "description": "Search file and directory paths within a repository and branch using a non-empty plain query string. Matching is case-insensitive substring matching on path text (single query string, not OR-token search). This is path-only matching and does not enumerate full directory contents; use file_list for enumeration and file_content for file bodies. Do not send filter syntax like `path:` or glob patterns here. Optional cursor/auto_paginate can fetch additional path matches. Includes freshness metadata.",
             "inputSchema": {
                 "type": "object",
                 "properties": {
                     "repo": { "type": "string" },
                     "branch": { "type": "string" },
-                    "query": { "type": "string", "description": "Plain fuzzy text only. Example: \"mcp serv\". Do not send `path:src/mcp` or glob syntax here." },
-                    "limit": { "type": "integer", "minimum": 1, "maximum": 50 }
+                    "query": { "type": "string", "description": "Plain query string only. Example: \"mcp_serv\". Matching is case-insensitive substring matching over path text. This is a single query string, not OR token search. Do not send `path:src/mcp` or glob syntax here." },
+                    "limit": { "type": "integer", "minimum": 1, "maximum": 50 },
+                    "cursor": { "type": "string", "description": "Optional pagination cursor from prior path_search response." },
+                    "auto_paginate": { "type": "boolean", "description": "If true, server attempts to return multiple pages up to max_pages/max_total_entries." },
+                    "max_pages": { "type": "integer", "minimum": 1, "maximum": 5 },
+                    "max_total_entries": { "type": "integer", "minimum": 1, "maximum": 200 }
                 },
                 "examples": [
-                    { "repo": "pointer", "branch": "main", "query": "mcp serv", "limit": 10 }
+                    { "repo": "pointer", "branch": "main", "query": "mcp_serv", "limit": 10 },
+                    { "repo": "pointer", "branch": "main", "query": "mcp_serv", "limit": 10, "cursor": "10" },
+                    { "repo": "pointer", "branch": "main", "query": "mcp_serv", "limit": 10, "auto_paginate": true, "max_pages": 3, "max_total_entries": 25 }
                 ],
                 "required": ["repo", "branch", "query"],
                 "additionalProperties": false
@@ -455,6 +467,7 @@ fn mcp_tools() -> Vec<Value> {
                             "language": { "type": "string" },
                             "scope": {
                                 "type": "string",
+                                "description": "Scope selector. Accepted values are case-insensitive.",
                                 "enum": ["repository", "directory", "file", "custom"]
                             },
                             "include_paths": {

src/mcp/tools.rs

@@ -21,6 +21,9 @@ use crate::services::search_service::search;
 const MAX_BATCH_QUERIES: usize = 8;
 const FILE_LIST_QUERY_TIMEOUT: Duration = Duration::from_secs(8);
 const FILE_LIST_MAX_SOURCE_ROWS: usize = 200_000;
+const MAX_AUTO_PAGES: u8 = 5;
+const MAX_AUTO_TOTAL_PATH_ENTRIES: usize = 200;
+const MAX_AUTO_TOTAL_FILE_LIST_ENTRIES: usize = 5000;
 
 pub async fn execute_search(payload: SearchToolRequest) -> Result<Value, String> {
     let request_echo = search_request_echo(&payload, None);
@@ -182,6 +185,14 @@ pub async fn execute_file_list(
     let root_path = normalize_repo_path(payload.path.unwrap_or_default());
     let requested_depth = payload.depth.clamp(1, 10);
     let limit = payload.limit.clamp(1, 5000);
+    let offset = parse_cursor_offset(payload.cursor.as_deref())?;
+    let auto_paginate = payload.auto_paginate.unwrap_or(false);
+    let max_pages = payload.max_pages.unwrap_or(1).clamp(1, MAX_AUTO_PAGES);
+    let max_total_entries = payload
+        .max_total_entries
+        .unwrap_or(limit)
+        .clamp(limit, MAX_AUTO_TOTAL_FILE_LIST_ENTRIES)
+        .min(limit.saturating_mul(max_pages as usize));
 
     let like_pattern = if root_path.is_empty() {
         "%".to_string()
@@ -225,11 +236,51 @@ pub async fn execute_file_list(
     }
 
     let mut entries = build_file_list_entries(&source_rows, &root_path, requested_depth);
-    if entries.len() > limit {
-        entries.truncate(limit);
+    let total_entries = entries.len();
+
+    if auto_paginate {
+        let slice_start = offset.min(total_entries);
+        let slice_len = max_total_entries.min(total_entries.saturating_sub(slice_start));
+        entries = entries
+            .into_iter()
+            .skip(slice_start)
+            .take(slice_len)
+            .collect();
+    } else {
+        let slice_start = offset.min(total_entries);
+        let slice_len = limit.min(total_entries.saturating_sub(slice_start));
+        entries = entries
+            .into_iter()
+            .skip(slice_start)
+            .take(slice_len)
+            .collect();
+    }
+
+    let visible_limit = if auto_paginate {
+        max_total_entries
+    } else {
+        limit
+    };
+    let has_more = offset.saturating_add(visible_limit) < total_entries;
+    let next_cursor = has_more.then(|| (offset + entries.len()).to_string());
+    let pages_fetched = if auto_paginate {
+        let per_page = limit.max(1);
+        let fetched = entries.len().div_ceil(per_page).max(1) as u8;
+        Some(fetched.min(max_pages))
+    } else {
+        None
+    };
+
+    if total_entries > limit {
         truncated = true;
         if truncated_reason.is_none() {
-            truncated_reason = Some("file_list reached entry limit".to_string());
+            let mut reason = "file_list reached entry limit".to_string();
+            if has_more {
+                reason.push_str("; narrow path/depth or continue with cursor");
+            } else {
+                reason.push_str("; narrow path/depth for fewer results");
+            }
+            truncated_reason = Some(reason);
         }
     }
 
@@ -244,6 +295,9 @@ pub async fn execute_file_list(
         requested_depth,
         truncated,
         truncated_reason,
+        has_more,
+        next_cursor,
+        pages_fetched,
         entries,
         index_freshness,
     })
@@ -257,24 +311,76 @@ pub async fn execute_path_search(
     }
     if looks_like_search_filter_query(&payload.query) {
         return Err(
-            "path_search query must be plain fuzzy text, not filter syntax such as repo: or path:"
+            "path_search query must be a plain query string, not filter syntax such as repo: or path:"
                 .to_string(),
         );
     }
     let repo = payload.repo.clone();
     let branch = payload.branch.clone();
-    let entries = search_repo_paths(repo.clone(), branch.clone(), payload.query, payload.limit)
-        .await
-        .map_err(|err| err.to_string())?;
+    let limit = payload.limit.unwrap_or(10).clamp(1, 50) as usize;
+    let offset = parse_cursor_offset(payload.cursor.as_deref())?;
+    let auto_paginate = payload.auto_paginate.unwrap_or(false);
+    let max_pages = payload.max_pages.unwrap_or(1).clamp(1, MAX_AUTO_PAGES);
+    let max_total_entries = (payload
+        .max_total_entries
+        .unwrap_or(limit as u16)
+        .clamp(limit as u16, MAX_AUTO_TOTAL_PATH_ENTRIES as u16)
+        as usize)
+        .min(limit.saturating_mul(max_pages as usize));
+
+    let target_count = if auto_paginate {
+        max_total_entries
+    } else {
+        limit
+    };
+    let fetch_limit = (offset + target_count + 1).min(MAX_AUTO_TOTAL_PATH_ENTRIES + 1);
+    let fetched = search_repo_paths(
+        repo.clone(),
+        branch.clone(),
+        payload.query,
+        Some(fetch_limit as u16),
+    )
+    .await
+    .map_err(|err| err.to_string())?;
+    let has_more = fetched.len() > offset + target_count;
+    let entries: Vec<_> = fetched
+        .into_iter()
+        .skip(offset)
+        .take(target_count)
+        .collect();
+    let next_cursor = has_more.then(|| (offset + entries.len()).to_string());
+    let pages_fetched = if auto_paginate {
+        let fetched_pages = entries.len().div_ceil(limit).max(1) as u8;
+        Some(fetched_pages.min(max_pages))
+    } else {
+        None
+    };
+
     let index_freshness = resolve_branch_freshness(&repo, &branch, None)
         .await
         .unwrap_or_else(|_| unknown_freshness());
     Ok(PathSearchToolResponse {
         entries,
+        has_more,
+        next_cursor,
+        pages_fetched,
         index_freshness,
     })
 }
 
+fn parse_cursor_offset(cursor: Option<&str>) -> Result<usize, String> {
+    let Some(cursor) = cursor else {
+        return Ok(0);
+    };
+    let trimmed = cursor.trim();
+    if trimmed.is_empty() {
+        return Ok(0);
+    }
+    trimmed
+        .parse::<usize>()
+        .map_err(|_| "invalid cursor; expected a numeric offset".to_string())
+}
+
 pub async fn execute_symbol_insights(
     payload: SymbolInsightsToolRequest,
 ) -> Result<SymbolInsightsToolResponse, String> {
@@ -322,7 +428,7 @@ pub fn normalize_tool_error(tool: &str, err: String) -> (String, String, Option<
             "file_list_invalid_params".to_string(),
             "file_list does not accept `query`".to_string(),
             Some(
-                "Use `path` as an exact directory prefix for file_list, or call path_search with a plain fuzzy `query`."
+                "Use `path` as an exact directory prefix for file_list, or call path_search with a plain query string (case-insensitive substring matching)."
                     .to_string(),
             ),
         );
@@ -365,12 +471,12 @@ pub fn normalize_tool_error(tool: &str, err: String) -> (String, String, Option<
             ),
         );
     }
-    if tool == "path_search" && lower.contains("plain fuzzy text") {
+    if tool == "path_search" && lower.contains("plain query string") {
         return (
             "path_search_invalid_query_syntax".to_string(),
             err,
             Some(
-                "Use plain fuzzy text like `mcp serv` for path_search. Use file_list.path for directory prefixes and search.path/search.file for filters."
+                "Use a plain query string like `mcp_serv` for path_search (case-insensitive substring matching). Use file_list.path for directory prefixes and search.path/search.file for filters."
                     .to_string(),
             ),
         );