{"description":"Check if major AI LLM crawlers are allowed on a website based on robots.txt","endpoints":{"GET /check":{"description":"Check AI crawler permissions for a website","example":"/check?url=https://example.com","parameters":{"url":"The website URL to check (required)"}},"GET /faq":{"description":"Generate 2-10 FAQs for a website based on homepage content.","example":"/faq?url=https://example.com","parameters":{"url":"The website URL to generate FAQs for (required)"}},"GET /health":{"description":"Health check endpoint"},"GET /llmstxt":{"description":"Generate an llms.txt file for a website. Scrapes homepage and sitemap, then uses AI to generate the file.","example":"/llmstxt?url=https://example.com","parameters":{"url":"The website URL to generate llms.txt for (required)"}},"POST /cluster":{"body":"CSV data with Column A = keyword, Column B = search volume","content_type":"text/csv","description":"Cluster keywords using AI. Uses free OpenRouter models with automatic retry.","example_csv":"keyword,search_volume\nbuy shoes online,12000\nbest running shoes,8500\nshoe store near me,5000"},"POST /humanize":{"body":"JSON with 'text' field, or plain text","description":"Transform AI-generated text into natural, human-like writing.","example_json":"{\"text\": \"Your AI-generated text here...\"}"}},"name":"AI Crawler Checker API","supported_bots":[{"agents":["GPTBot","ChatGPT-User"],"company":"OpenAI","name":"ChatGPT"},{"agents":["anthropic-ai","ClaudeBot","Claude-Web"],"company":"Anthropic","name":"Claude"},{"agents":["Google-Extended","Googlebot"],"company":"Google","name":"Gemini"},{"agents":["PerplexityBot"],"company":"Perplexity AI","name":"Perplexity"},{"agents":["Amazonbot"],"company":"Amazon","name":"Amazon AI"},{"agents":["FacebookBot","FacebookExternalHit","meta-externalagent"],"company":"Meta","name":"Meta AI"},{"agents":["cohere-ai"],"company":"Cohere","name":"Cohere"},{"agents":["CCBot"],"company":"Common Crawl","name":"Common Crawl"},{"agents":["Bytespider"],"company":"ByteDance","name":"ByteDance AI"},{"agents":["Applebot-Extended"],"company":"Apple","name":"Apple AI"}],"version":"1.0.0"}
