This implementation allows your local LLM (for example running on LM Studio) to utilize unclecode/crawl4ai to get informatino from URLs!
pip install mcp crawl4ai
playwright installmpc.json in your favourite LLM inference software{
"mcpServers": {
"crawl4ai": {
"command": "python",
"args": [
"<path_to_file>/crawl_mcp_server.py"
]
}
}
}| import sys | |
| # MCP stdio_server reads sys.stdout.buffer for the real fd. | |
| # crawl4ai pollutes sys.stdout with print() calls which corrupt the protocol. | |
| # Fix: keep .buffer pointing to real stdout (for MCP), but route text writes to stderr. | |
| class _StderrStdout: | |
| def __init__(self): | |
| self.buffer = sys.stdout.buffer # MCP uses this — must stay as real stdout | |
| def write(self, text): | |
| sys.stderr.write(text) | |
| def flush(self): | |
| sys.stderr.flush() | |
| sys.stdout = _StderrStdout() | |
| from mcp.server import Server | |
| from mcp.server.stdio import stdio_server | |
| from mcp.types import Tool, TextContent | |
| import asyncio | |
| from crawl4ai import AsyncWebCrawler | |
| app = Server("crawl4ai") | |
| @app.list_tools() | |
| async def list_tools(): | |
| return [Tool( | |
| name="crawl_url", | |
| description="Fetch URL content", | |
| inputSchema={"type": "object", "properties": {"url": {"type": "string"}}} | |
| )] | |
| @app.call_tool() | |
| async def call_tool(name, arguments): | |
| if name == "crawl_url": | |
| try: | |
| async with AsyncWebCrawler(verbose=False) as crawler: | |
| result = await crawler.arun(url=arguments["url"]) | |
| content = getattr(result, 'markdown', str(result)) or "" | |
| return [TextContent(type="text", text=content)] | |
| except Exception as e: | |
| return [TextContent(type="text", text=f"Error crawling URL: {e}")] | |
| raise ValueError(f"Unknown tool: {name}") | |
| async def main(): | |
| async with stdio_server() as (read_stream, write_stream): | |
| await app.run(read_stream, write_stream, app.create_initialization_options()) | |
| if __name__ == "__main__": | |
| asyncio.run(main()) |