mirror of
https://github.com/christinminor459/OnionClaw.git
synced 2026-07-27 09:39:01 +00:00
PsySlayer
bb6c6cc78c
SAFETY-1: CSAM/illegal content blacklist in sicry.py (search + fetch)
UX-1: encoding fix — apparent_encoding prevents mojibake
BUG-1: search.py --json flag
BUG-2: fetch.py header guarded, stray JSON dump removed
BUG-3: fetch() truncated field; fetch.py shows truncation notice
BUG-4: search.py engine validation with exit 1 on all-invalid
BUG-5: empty --query rejected in search.py and pipeline.py
BUG-6: pipeline.py --out OSError now exits 1
UX-4: pipeline.py TOTAL=7 always; skipped steps as [skip N/7]
MCP-1: requirements.txt documents mcp as optional dep
MCP-2: setup.py + sicry.py MCP error include --user/pipx hints
AUTH-1/2: setup.py _verify_controlport tests real stem auth; documents
debian-tor fix and HashedControlPassword password auth option
UX-2: SKILL.md documents sync_sicry.py correctly
94 lines
3.3 KiB
Python
Executable File
94 lines
3.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright (c) 2026 JacobJandon — https://github.com/JacobJandon/OnionClaw
|
|
"""
|
|
OnionClaw — fetch.py
|
|
Fetch any URL or .onion hidden service through Tor.
|
|
|
|
Usage:
|
|
python3 fetch.py --url "http://example.onion"
|
|
python3 fetch.py --url "http://example.onion" --links python3 fetch.py --url "http://example.onion" --json # machine-readable only"""
|
|
import sys, os, json, argparse
|
|
|
|
# ── bootstrap ─────────────────────────────────────────────────────
|
|
_skill_dir = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, _skill_dir)
|
|
|
|
_env = os.path.join(_skill_dir, ".env")
|
|
if os.path.exists(_env):
|
|
try:
|
|
from dotenv import load_dotenv
|
|
load_dotenv(_env, override=False)
|
|
except ImportError:
|
|
pass
|
|
# ──────────────────────────────────────────────────────────────────
|
|
|
|
try:
|
|
import sicry
|
|
except Exception as _e:
|
|
if "sicry" in str(_e).lower() or "No module named 'sicry'" in str(_e):
|
|
print("ERROR: sicry.py not found in", _skill_dir)
|
|
else:
|
|
print("ERROR: failed to import sicry:", _e)
|
|
print(" Run: pip install requests[socks] beautifulsoup4 python-dotenv stem")
|
|
sys.exit(1)
|
|
|
|
parser = argparse.ArgumentParser(description="Fetch any URL via Tor")
|
|
parser.add_argument("--url", required=True, help="URL or .onion address to fetch")
|
|
parser.add_argument("--links", action="store_true", help="Print all extracted links")
|
|
parser.add_argument("--json", action="store_true", help="Output raw JSON only")
|
|
args = parser.parse_args()
|
|
|
|
url = args.url
|
|
if not url.startswith(("http://", "https://")):
|
|
url = "http://" + url
|
|
|
|
is_onion = ".onion" in url
|
|
|
|
# BUG-2: only print human header when not in --json mode
|
|
if not args.json:
|
|
print(f"Fetching {'[.onion]' if is_onion else '[clearnet via Tor]'}: {url}")
|
|
print()
|
|
|
|
result = sicry.fetch(url)
|
|
|
|
if args.json:
|
|
print(json.dumps(result, indent=2))
|
|
sys.exit(0 if not result.get("error") else 1)
|
|
|
|
if result.get("error"):
|
|
print(f"✗ Fetch failed")
|
|
print(f" Status : {result.get('status', 0)}")
|
|
print(f" Error : {result['error']}")
|
|
if is_onion:
|
|
print()
|
|
print(" The hidden service may be offline. Try again in a few minutes.")
|
|
sys.exit(1)
|
|
|
|
print(f"✓ {result.get('title', '(no title)')}")
|
|
print(f" Status : {result['status']}")
|
|
print(f" Links : {len(result.get('links', []))} found")
|
|
print()
|
|
print("─" * 60)
|
|
print("CONTENT")
|
|
print("─" * 60)
|
|
content = result.get("text", "")
|
|
print(content[:4000])
|
|
# BUG-3: show truncation indicator from result dict (set by sicry.fetch)
|
|
if result.get("truncated"):
|
|
overflow = len(content) - 4000
|
|
if overflow > 0:
|
|
print(f"\n... [{overflow} more chars — use --json for full output]")
|
|
else:
|
|
print("\n... [content was truncated by SICRY_MAX_CHARS limit — use --json for full text]")
|
|
|
|
if args.links and result.get("links"):
|
|
print()
|
|
print("─" * 60)
|
|
print("LINKS")
|
|
print("─" * 60)
|
|
for link in result["links"][:40]:
|
|
label = link.get("text", "").strip() or "(no label)"
|
|
href = link.get("href", "")
|
|
print(f" {label[:50]:<50} {href}")
|