Update agent/tools/web_tools.py

agent/tools/web_tools.py

@@ -1,5 +1,5 @@
 """
-NeuraPrompt Agent — Web Tools v8.0 (Multi-Engine Fallback, No API Keys)
+NeuraPrompt Agent — Web Tools v8.1 (Faster Failover + Resilience)
 """
 
 import requests
@@ -12,16 +12,16 @@ from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 import logging
 
-log = logging.getLogger("agent.tools.web.v8.0")
+log = logging.getLogger("agent.tools.web.v8.1")
 
 # ─────────────────────────────────────────────────────────────
-# CONFIG
+# CONFIG - TUNED FOR SPEED & RELIABILITY
 # ─────────────────────────────────────────────────────────────
 
-TIMEOUT_SEARCH = 12
-TIMEOUT_FETCH = 20
-MAX_RETRIES = 3
-BACKOFF_FACTOR = 1.5
+TIMEOUT_SEARCH = 8
+TIMEOUT_FETCH = 15
+MAX_RETRIES = 1          # Reduced → fail fast
+BACKOFF_FACTOR = 0.3     # Minimal backoff
 MAX_RESULTS = 6
 
 USER_AGENTS = [
@@ -32,10 +32,13 @@ USER_AGENTS = [
 ]
 
 SEARXNG_INSTANCES = [
-    "https://search.sapti.me",
     "https://searx.be",
+    "https://search.ononoki.org",
     "https://searx.tiekoetter.com",
     "https://searx.prvcy.eu",
+    "https://search.sapti.me",
+    "https://darmarit.org/searx",
+    "https://searxng.site",
 ]
 
 
@@ -140,7 +143,6 @@ def _search_ddg_lite(session: requests.Session, query: str) -> list[dict] | None
     soup = BeautifulSoup(response.text, "lxml")
     results = []
     
-    # DDG Lite uses table.result rows
     for tr in soup.select("table.result")[:MAX_RESULTS]:
         link = tr.select_one("a.result-link")
         if not link:
@@ -148,7 +150,6 @@ def _search_ddg_lite(session: requests.Session, query: str) -> list[dict] | None
             
         title = link.get_text(strip=True)
         href = link.get("href", "")
-        # DDG Lite sometimes uses relative or redirect URLs
         if href.startswith("/"):
             href = f"https://lite.duckduckgo.com{href}"
         elif href.startswith("//"):
@@ -187,7 +188,6 @@ def _search_ddg_api(session: requests.Session, query: str) -> list[dict] | None:
     
     results = []
     
-    # Main abstract answer
     abstract = (data.get("AbstractText") or data.get("Answer") or "").strip()
     abstract_url = data.get("AbstractURL", "")
     if abstract and abstract_url:
@@ -198,7 +198,6 @@ def _search_ddg_api(session: requests.Session, query: str) -> list[dict] | None:
             "engine": "ddg-api"
         })
     
-    # Related topics
     for topic in data.get("RelatedTopics", [])[:MAX_RESULTS - 1]:
         text = topic.get("Text", "").strip()
         first_url = topic.get("FirstURL", "")
@@ -214,7 +213,7 @@ def _search_ddg_api(session: requests.Session, query: str) -> list[dict] | None:
 
 
 def _search_bing(session: requests.Session, query: str) -> list[dict] | None:
-    """Engine 3: Bing HTML (moderate blocking, rich results)."""
+    """Engine 3: Bing HTML."""
     url = f"https://www.bing.com/search?q={quote_plus(query)}"
     
     response = _safe_request(
@@ -229,7 +228,6 @@ def _search_bing(session: requests.Session, query: str) -> list[dict] | None:
     soup = BeautifulSoup(response.text, "lxml")
     results = []
     
-    # Bing uses li.b_algo for organic results
     for li in soup.select("li.b_algo")[:MAX_RESULTS]:
         a = li.select_one("a")
         if not a:
@@ -238,7 +236,6 @@ def _search_bing(session: requests.Session, query: str) -> list[dict] | None:
         title = a.get_text(strip=True)
         href = a.get("href", "")
         
-        # Extract snippet from various Bing structures
         snippet = ""
         for sel in ["p", ".b_caption p", "div.b_attribution+div", ".b_snippet"]:
             el = li.select_one(sel)
@@ -258,10 +255,10 @@ def _search_bing(session: requests.Session, query: str) -> list[dict] | None:
 
 
 def _search_searxng(session: requests.Session, query: str) -> list[dict] | None:
-    """Engine 4: SearXNG public instances (meta-search, JSON output)."""
+    """Engine 4: SearXNG public instances."""
     random.shuffle(SEARXNG_INSTANCES)
     
-    for base in SEARXNG_INSTANCES[:3]:  # Try 3 random instances
+    for base in SEARXNG_INSTANCES[:3]:
         url = f"{base}/search?q={quote_plus(query)}&format=json&language=en"
         
         response = _safe_request(
@@ -304,7 +301,6 @@ def _search_searxng(session: requests.Session, query: str) -> list[dict] | None:
 def web_search(query: str) -> str:
     """
     Search the web using multiple no-key engines with automatic fallback.
-    
     Priority: DDG Lite → DDG API → Bing → SearXNG
     """
     if not query or not query.strip():
@@ -381,22 +377,18 @@ def fetch_url(url: str) -> str:
             "Tip: Try a different URL or check if the site requires JavaScript."
         )
     
-    # Handle non-HTML content
     content_type = response.headers.get("content-type", "").lower()
     if "text/html" not in content_type:
         preview = response.text[:8000]
         return f"[Non-HTML content: {content_type}]\n\n{preview}"
     
-    # Parse and clean HTML
     soup = BeautifulSoup(response.text, "lxml")
     
-    # Remove noise elements
     for tag in soup(["script", "style", "nav", "header", "footer", "aside", 
                      "form", "iframe", "noscript", "svg", "canvas", 
                      "advertisement", ".ad", ".ads", ".cookie-banner"]):
         tag.decompose()
     
-    # Strategy 1: Look for semantic content containers
     content_blocks = []
     for selector in ["article", "main", "[role='main']", ".content", ".post", ".entry"]:
         for el in soup.select(selector):
@@ -404,18 +396,15 @@ def fetch_url(url: str) -> str:
             if len(text) > 300:
                 content_blocks.append(text)
     
-    # Strategy 2: Fallback to headings + paragraphs
     if not content_blocks:
         for tag in soup.find_all(["h1", "h2", "h3", "h4", "p", "li", "td"]):
             text = tag.get_text(" ", strip=True)
             if len(text) > 30:
                 content_blocks.append(text)
     
-    # Deduplicate and join
     seen = set()
     final_blocks = []
     for block in content_blocks:
-        # Simple dedup: skip if very similar to already-seen
         sig = block[:100].lower()
         if sig not in seen:
             seen.add(sig)
@@ -426,7 +415,6 @@ def fetch_url(url: str) -> str:
     if not text:
         return "No readable content found. The page may be JavaScript-rendered or heavily obfuscated."
     
-    # Add metadata header
     title = ""
     title_tag = soup.find("title")
     if title_tag:
@@ -442,7 +430,8 @@ def fetch_url(url: str) -> str:
 # ─────────────────────────────────────────────────────────────
 
 if __name__ == "__main__":
-    # Quick self-test
+    import logging
+    logging.basicConfig(level=logging.INFO)
     print("=" * 60)
     print("TEST: web_search('Python programming language')")
     print("=" * 60)
@@ -450,5 +439,4 @@ if __name__ == "__main__":
     print("\n" + "=" * 60)
     print("TEST: fetch_url('https://en.wikipedia.org/wiki/Python_(programming_language)')")
     print("=" * 60)
-    print(fetch_url("https://en.wikipedia.org/wiki/Python_(programming_language)")[:1500])
-    
+    print(fetch_url("https://en.wikipedia.org/wiki/Python_(programming_language)")[:1500])