Spaces:
Sleeping
Sleeping
SlipStream90
Fix Groq 413 TPM: trim tool outputs (3500/3000 chars), shorten prompt, reduce max_tokens 512, add rate-limit retry
0c74cb3 Download tools.py from SlipStream33/Course_Check: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/spaces/SlipStream33/Course_Check/resolve/main/tools.py
- Command line
-
hf download hf://spaces/SlipStream33/Course_Check/tools.py
-
curl -L -o tools.py https://huggingface.co/spaces/SlipStream33/Course_Check/resolve/main/tools.py
12.6 kB
| """Tools for the GAIA agent: web search, webpage reading, Wikipedia, | |
| math, file analysis (Excel/CSV/PDF/text/images/audio), and YouTube transcripts. | |
| Every tool catches its own exceptions and returns an "ERROR: ..." string on | |
| failure so the agent loop never crashes mid-run. | |
| """ | |
| import base64 | |
| import io | |
| import json | |
| import os | |
| import re | |
| import zipfile | |
| from pathlib import Path | |
| import html2text | |
| import pandas as pd | |
| import requests | |
| from langchain_core.tools import tool | |
| WORKSPACE_DIR = Path(os.getenv("AGENT_WORKSPACE", "workspace")) | |
| USER_AGENT = "Mozilla/5.0 (compatible; gaia-course-agent/1.0)" | |
| # Groq free tier is 8000 TPM — keep tool outputs small to stay under limit | |
| WEBPAGE_MAX_CHARS = 3500 | |
| FILE_TEXT_MAX_CHARS = 3000 | |
| # ---------------------------------------------------------------- search --- | |
| def web_search(query: str) -> str: | |
| """Search the web. Returns top results with title/URL/snippet.""" | |
| try: | |
| from ddgs import DDGS | |
| results = DDGS().text(query, max_results=5) | |
| if not results: | |
| return f"No results found for '{query}'." | |
| lines = [] | |
| for i, r in enumerate(results, 1): | |
| lines.append( | |
| f"{i}. {r.get('title', '')}\n URL: {r.get('href', '')}\n {r.get('body', '')}" | |
| ) | |
| return "\n\n".join(lines) | |
| except Exception as e: | |
| return f"ERROR: web_search failed: {e}" | |
| def visit_webpage(url: str) -> str: | |
| """Fetch a webpage and return its readable text content (markdown). | |
| Use after web_search when a snippet is not enough.""" | |
| try: | |
| resp = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=20) | |
| resp.raise_for_status() | |
| content_type = resp.headers.get("Content-Type", "") | |
| if "html" in content_type or "xml" in content_type or not content_type: | |
| converter = html2text.HTML2Text() | |
| converter.ignore_links = True | |
| converter.ignore_images = True | |
| text = converter.handle(resp.text) | |
| else: | |
| text = resp.text | |
| text = re.sub(r"\n{3,}", "\n\n", text).strip() | |
| if len(text) > WEBPAGE_MAX_CHARS: | |
| text = text[:WEBPAGE_MAX_CHARS] + f"\n... [truncated, {len(text)} chars total]" | |
| return text or "Page returned empty content." | |
| except Exception as e: | |
| return f"ERROR: visit_webpage failed for {url}: {e}" | |
| def wikipedia_lookup(query: str) -> str: | |
| """Search English Wikipedia and return the summary of the best-matching | |
| article. Good for factual questions about people, albums, events.""" | |
| try: | |
| headers = {"User-Agent": USER_AGENT} | |
| search_resp = requests.get( | |
| "https://en.wikipedia.org/w/api.php", | |
| params={ | |
| "action": "query", "list": "search", "srsearch": query, | |
| "format": "json", "srlimit": 5, | |
| }, | |
| headers=headers, timeout=20, | |
| ) | |
| search_resp.raise_for_status() | |
| hits = search_resp.json().get("query", {}).get("search", []) | |
| if not hits: | |
| return f"No Wikipedia article found for '{query}'." | |
| title = hits[0]["title"] | |
| sum_resp = requests.get( | |
| f"https://en.wikipedia.org/api/rest_v1/page/summary/{requests.utils.quote(title)}", | |
| headers=headers, timeout=20, | |
| ) | |
| sum_resp.raise_for_status() | |
| data = sum_resp.json() | |
| summary = data.get("extract", "") | |
| if len(summary) > FILE_TEXT_MAX_CHARS: | |
| summary = summary[:FILE_TEXT_MAX_CHARS] + "... [truncated]" | |
| other = [h["title"] for h in hits[1:4]] | |
| out = f"Wikipedia article '{data.get('title', title)}':\n{summary}" | |
| if other: | |
| out += f"\n(Other matches: {other})" | |
| return out | |
| except Exception as e: | |
| return f"ERROR: wikipedia_lookup failed: {e}" | |
| # ------------------------------------------------------------------ math --- | |
| def calculate(expression: str) -> str: | |
| """Evaluate a mathematical expression exactly (supports arithmetic, | |
| powers, roots, percentages, factorial via sympy syntax like sqrt(2)).""" | |
| try: | |
| from sympy import parse_expr | |
| expr = parse_expr( | |
| expression, | |
| transformations="all", | |
| evaluate=True, | |
| ) | |
| try: | |
| num = float(expr.evalf(15)) | |
| if abs(num - round(num)) < 1e-9: | |
| return str(int(round(num))) | |
| return repr(num) | |
| except (TypeError, ValueError): | |
| # Symbolic (non-numeric) result | |
| return str(expr) | |
| except Exception as e: | |
| return f"ERROR: calculate failed for '{expression}': {e}" | |
| # ----------------------------------------------------------------- files --- | |
| def _excel_report(path: Path) -> str: | |
| sheets = pd.read_excel(path, sheet_name=None) | |
| parts = [] | |
| for name, df in sheets.items(): | |
| head = df.head(10).to_string(index=False) | |
| parts.append( | |
| f"Sheet '{name}': {df.shape[0]} rows x {df.shape[1]} cols\n" | |
| f"Columns: {list(df.columns)}\nData types:\n{df.dtypes.to_string()}\n" | |
| f"First rows:\n{head}" | |
| ) | |
| return "\n\n".join(parts) | |
| def _pdf_report(path: Path) -> str: | |
| from pypdf import PdfReader | |
| reader = PdfReader(str(path)) | |
| pages = [] | |
| total = 0 | |
| for i, p in enumerate(reader.pages): | |
| t = (p.extract_text() or "").strip() | |
| pages.append(f"--- Page {i + 1} ---\n{t}") | |
| total += len(t) | |
| if total > FILE_TEXT_MAX_CHARS * 2: | |
| break | |
| text = "\n".join(pages) | |
| if len(text) > FILE_TEXT_MAX_CHARS: | |
| text = text[:FILE_TEXT_MAX_CHARS] + "... [truncated]" | |
| return f"PDF with {len(reader.pages)} pages:\n{text}" | |
| def _docx_report(path: Path) -> str: | |
| import docx | |
| d = docx.Document(str(path)) | |
| paras = [p.text for p in d.paragraphs if p.text.strip()] | |
| text = "\n".join(paras) | |
| if len(text) > FILE_TEXT_MAX_CHARS: | |
| text = text[:FILE_TEXT_MAX_CHARS] + "... [truncated]" | |
| return f"DOCX document ({len(paras)} paragraphs):\n{text}" | |
| def _zip_report(path: Path) -> str: | |
| out_dir = WORKSPACE_DIR / path.stem | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| with zipfile.ZipFile(path) as zf: | |
| names = zf.namelist() | |
| zf.extractall(out_dir) | |
| listing = "\n".join(names[:30]) | |
| more = f"\n... and {len(names) - 30} more" if len(names) > 30 else "" | |
| return ( | |
| f"ZIP archive extracted to {out_dir}. Contents:\n{listing}{more}\n" | |
| f"Use analyze_file on individual extracted files to inspect them." | |
| ) | |
| def _image_report(path: Path) -> str: | |
| from PIL import Image | |
| img = Image.open(path) | |
| meta = f"Image: format={img.format}, size={img.size}, mode={img.mode}" | |
| desc = _describe_image_with_vlm(path) | |
| if desc: | |
| return f"{meta}\nVisual description:\n{desc}" | |
| return meta | |
| def _describe_image_with_vlm(path: Path) -> str | None: | |
| token = os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN") | |
| if not token: | |
| return None | |
| try: | |
| from huggingface_hub import InferenceClient | |
| client = InferenceClient(token=token) | |
| b64 = base64.b64encode(Path(path).read_bytes()).decode() | |
| model = os.getenv("HF_VLM_MODEL", "Qwen/Qwen2-VL-72B-Instruct") | |
| resp = client.chat_completion( | |
| model=model, | |
| messages=[{ | |
| "role": "user", | |
| "content": [ | |
| {"type": "image_url", | |
| "image_url": {"url": f"data:image;base64,{b64}"}}, | |
| {"type": "text", | |
| "text": "Describe this image factually and in detail " | |
| "(objects, text visible, counts, colors)."}, | |
| ], | |
| }], | |
| max_tokens=400, | |
| ) | |
| return resp.choices[0].message.content | |
| except Exception as e: | |
| return f"(vision model unavailable: {e})" | |
| def _audio_report(path: Path) -> str: | |
| token = os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN") | |
| if not token: | |
| return f"Audio file {path.name}: transcription requires HF_TOKEN." | |
| try: | |
| from huggingface_hub import InferenceClient | |
| client = InferenceClient(token=token) | |
| model = os.getenv("HF_ASR_MODEL", "facebook/whisper-large-v3") | |
| out = client.automatic_speech_recognition(str(path), model=model) | |
| text = getattr(out, "text", None) or str(out) | |
| return f"Audio transcript of {path.name}:\n{text}" | |
| except Exception as e: | |
| return f"Audio file {path.name}; transcription failed: {e}" | |
| def analyze_file(path: str) -> str: | |
| """Inspect a local file and return its contents or structure. | |
| Handles Excel, CSV, JSON, text, PDF, DOCX, ZIP (extracts it), | |
| images (metadata + visual description), and audio (transcript). | |
| Always call this first when given a file path.""" | |
| p = Path(path) | |
| if not p.exists(): | |
| candidates = list(WORKSPACE_DIR.rglob(p.name)) | |
| if candidates: | |
| p = candidates[0] | |
| else: | |
| return f"ERROR: file not found: {path}" | |
| ext = p.suffix.lower().lstrip(".") | |
| try: | |
| if ext in ("xlsx", "xls"): | |
| return _excel_report(p) | |
| if ext in ("csv", "tsv"): | |
| sep = "\t" if ext == "tsv" else "," | |
| df = pd.read_csv(p, sep=sep) | |
| return ( | |
| f"CSV: {df.shape[0]} rows x {df.shape[1]} cols\n" | |
| f"Columns: {list(df.columns)}\nFirst rows:\n{df.head(10).to_string(index=False)}" | |
| ) | |
| if ext == "json": | |
| data = json.loads(p.read_text(encoding="utf-8", errors="replace")) | |
| text = json.dumps(data, indent=2, ensure_ascii=False) | |
| return text[:FILE_TEXT_MAX_CHARS] | |
| if ext == "pdf": | |
| return _pdf_report(p) | |
| if ext == "docx": | |
| return _docx_report(p) | |
| if ext == "zip": | |
| return _zip_report(p) | |
| if ext in ("png", "jpg", "jpeg", "webp", "gif", "bmp", "tiff"): | |
| return _image_report(p) | |
| if ext in ("mp3", "wav", "m4a", "flac", "ogg"): | |
| return _audio_report(p) | |
| # Treat everything else as text | |
| text = p.read_text(encoding="utf-8", errors="replace") | |
| if len(text) > FILE_TEXT_MAX_CHARS: | |
| text = text[:FILE_TEXT_MAX_CHARS] + "... [truncated]" | |
| return f"Text file ({ext or 'no extension'}):\n{text}" | |
| except Exception as e: | |
| return f"ERROR: analyze_file failed on {p.name}: {e}" | |
| def run_python(code: str) -> str: | |
| """Execute short Python code for data analysis on downloaded files. | |
| print() anything you want to see; its output is returned to you. | |
| Available: pd (pandas), np (numpy), Path, and all standard library. | |
| Keep code under ~50 lines.""" | |
| import contextlib | |
| import numpy as np | |
| buf = io.StringIO() | |
| safe_globals = { | |
| "__builtins__": __builtins__, | |
| "pd": pd, | |
| "np": np, | |
| "Path": Path, | |
| "json": json, | |
| "re": re, | |
| "base64": base64, | |
| "WORKSPACE_DIR": str(WORKSPACE_DIR), | |
| } | |
| try: | |
| with contextlib.redirect_stdout(buf): | |
| exec(code, safe_globals) | |
| out = buf.getvalue().strip() | |
| return out if out else "(code ran, no output printed)" | |
| except Exception as e: | |
| partial = buf.getvalue().strip() | |
| err = f"ERROR in python code: {type(e).__name__}: {e}" | |
| return f"{partial}\n{err}" if partial else err | |
| # ------------------------------------------------------------- youtube ---- | |
| def youtube_transcript(url_or_id: str) -> str: | |
| """Get the transcript of a YouTube video. Pass a full URL or video ID.""" | |
| try: | |
| from youtube_transcript_api import YouTubeTranscriptApi | |
| m = re.search(r"(?:v=|youtu\.be/|shorts/|embed/)([\w-]{11})", url_or_id) | |
| video_id = m.group(1) if m else url_or_id.strip() | |
| api = YouTubeTranscriptApi() | |
| fetched = api.fetch(video_id) | |
| try: | |
| raw = fetched.to_raw_data() | |
| except AttributeError: | |
| raw = [{"text": s.text} for s in fetched] | |
| text = " ".join(seg["text"] for seg in raw) | |
| if len(text) > FILE_TEXT_MAX_CHARS: | |
| text = text[:FILE_TEXT_MAX_CHARS] + "... [truncated]" | |
| return f"YouTube transcript ({video_id}):\n{text}" | |
| except Exception as e: | |
| return f"ERROR: youtube_transcript failed: {e}" | |
| ALL_TOOLS = [ | |
| web_search, | |
| visit_webpage, | |
| wikipedia_lookup, | |
| calculate, | |
| analyze_file, | |
| run_python, | |
| youtube_transcript, | |
| ] | |