# Prevent all bots from accessing the entire site
User-agent: *
Disallow: /

# Specific disallow rules for known scraping bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Hunter.io
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: SimilartechBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: MeltwaterBot
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: SiteimproveBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: wget
Disallow: /

User-agent: curl
Disallow: /

# Search Engine Bots
# Google:
User-agent: Googlebot
Disallow: /

User-agent: Googlebot-Image
Disallow: /

User-agent: Googlebot-News
Disallow: /

User-agent: Googlebot-Video
Disallow: /

User-agent: Googlebot-Mobile
Disallow: /

User-agent: AdsBot-Google
Disallow: /

User-agent: Storebot-Google
Disallow: /

User-agent: Google-InspectionTool
Disallow: /

User-agent: GoogleOther
Disallow: /

# Bing:
User-agent: Bingbot
Disallow: /

User-agent: msnbot
Disallow: /

# DuckDuckGo:
User-agent: DuckDuckBot
Disallow: /

# Baidu:
User-agent: Baiduspider
Disallow: /

# Yandex:
User-agent: YandexBot
Disallow: /

# Apple:
User-agent: Applebot
Disallow: /

# Social Media Bots: 

# Facebook:
User-agent: facebookexternalhit
Disallow: /

User-agent: Facebot
Disallow: /

# Twitter:
 User-agent: Twitterbot
Disallow: /

# LinkedIn:
User-agent: LinkedInBot
Disallow: /

# SEO and Analytics Bots:

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent:  Mozbot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: Majestic-EightyBot
Disallow: /

User-agent: UptimeRobot
Disallow: /

# AI and LLM Crawlers: 

User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: .ai 
Disallow: /

User-agent: Agentic
Disallow: /

User-agent: AI Article Writer
Disallow: /

User-agent: AI Content Detector
Disallow: /

User-agent: AI Dungeon
Disallow: /

User-agent: AI Search Engine
Disallow: /

User-agent: AI SEO Crawler
Disallow: /

User-agent: AI Writer
Disallow: /

User-agent: AI21 Labs
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: AIBot
Disallow: /

User-agent: AIMatrix
Disallow: /

User-agent: AISearchBot
Disallow: /

User-agent: AI Training
Disallow: /

User-agent: AITraining
Disallow: /

User-agent: Alexa
Disallow: /

User-agent: Alpha AI
Disallow: /

User-agent: AlphaAI
Disallow: /

User-agent: Amazon Bedrock
Disallow: /

User-agent: Amazon-Kendra
Disallow: /

User-agent: Amazon Lex
Disallow: /

User-agent: Amazon Comprehend
Disallow: /

User-agent: Amazon Sagemaker
Disallow: /

User-agent: Amazon Silk
Disallow: /

User-agent: Amazon Textract
Disallow: /

User-agent: AmazonBot
Disallow: /

User-agent: Amelia
Disallow: /

User-agent: AndersPinkBot
Disallow: /

User-agent: Anthropic
Disallow: /

User-agent: AnyPicker
Disallow: /

User-agent: Anyword
Disallow: /

User-agent: Aria Browse
Disallow: /

User-agent: Articoolo
Disallow: /

User-agent: Automated Writer
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: Azure
Disallow: /

User-agent: BardBot
Disallow: /

User-agent: Brave Leo
Disallow: /

User-agent: ByteDance
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CatBoost
Disallow: /

User-agent: CC-Crawler
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGLM
Disallow: /

User-agent: Chinchilla
Disallow: /

User-agent: Claude
Disallow: /

User-agent: ClearScope
Disallow: /

User-agent: Cohere
Disallow: /

User-agent: Common Crawl
Disallow: /

User-agent: CommonCrawl
Disallow: /

User-agent: Content Harmony
Disallow: /

User-agent: Content King
Disallow: /

User-agent: Content Optimizer
Disallow: /

User-agent: Content Samurai
Disallow: /

User-agent: ContentAtScale
Disallow: /

User-agent: ContentBot
Disallow: /

User-agent: Contentedge
Disallow: /

User-agent: Conversion AI
Disallow: /

User-agent: Copilot
Disallow: /

User-agent: CopyAI
Disallow: /

User-agent: Copymatic
Disallow: /

User-agent: Copyscape
Disallow: /

User-agent: Cotoyogi
Disallow: /

User-agent: CrawlQ AI
Disallow: /

User-agent: Crawlspace
Disallow: /

User-agent: Crew AI
Disallow: /

User-agent: CrewAI
Disallow: /

User-agent: DALL-E
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: DataProvider
Disallow: /

User-agent: DeepAI
Disallow: /

User-agent: DeepL
Disallow: /

User-agent: DeepMind
Disallow: /

User-agent: DeepSeek
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Doubao AI
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: FacebookExternalHit
Disallow: /

User-agent: Falcon
Disallow: /

User-agent: Firecrawl
Disallow: /

User-agent: Flyriver
Disallow: /

User-agent: Frase AI
Disallow: /

User-agent: FriendlyCrawler
Disallow: /

User-agent: Gemini
Disallow: /

User-agent: Gemma
Disallow: /

User-agent: GenAI
Disallow: /

User-agent: Genspark
Disallow: /

User-agent: GLM
Disallow: /

User-agent: Goose
Disallow: /

User-agent: GPT
Disallow: /

User-agent: Grammarly
Disallow: /

User-agent: Grendizer
Disallow: /

User-agent: Grok
Disallow: /

User-agent: GT Bot
Disallow: /

User-agent: GTBot
Disallow: /

User-agent: Hemingway Editor
Disallow: /

User-agent: Hugging Face
Disallow: /

User-agent: Hypotenuse AI
Disallow: /

User-agent: iaskspider
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: ImageGen
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: INK Editor
Disallow: /

User-agent: INKforall
Disallow: /

User-agent: IntelliSeek
Disallow: /

User-agent: Inferkit
Disallow: /

User-agent: ISSCyberRiskCrawler
Disallow: /

User-agent: JasperAI
Disallow: /

User-agent: Kafkai
Disallow: /

User-agent: Kangaroo
Disallow: /

User-agent: Keyword Density AI
Disallow: /

User-agent: Knowledge
Disallow: /

User-agent: KomoBot
Disallow: /

User-agent: LLaMA
Disallow: /

User-agent: LLMs
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: MarketMuse
Disallow: /

User-agent: Meltwater
Disallow: /

User-agent: Meta AI
Disallow: /

User-agent: Meta-External
Disallow: /

User-agent: MetaAI
Disallow: /

User-agent: MetaTagBot
Disallow: /

User-agent: Mistral
Disallow: /

User-agent: Narrative
Disallow: /

User-agent: NeevaBot
Disallow: /

User-agent: Neural Text
Disallow: /

User-agent: NeuralSEO
Disallow: /

User-agent: Nova Act
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Open AI
Disallow: /

User-agent: OpenAI
Disallow: /

User-agent: OpenBot
Disallow: /

User-agent: OpenText AI
Disallow: /

User-agent: Operator
Disallow: /

User-agent: Outwrite
Disallow: /

User-agent: Page Analyzer AI
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Paperlibot
Disallow: /

User-agent: Paraphraser.io
Disallow: /

User-agent: Perplexity
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Phindbot
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: ProWritingAid
Disallow: /

User-agent: QuillBot
Disallow: /

User-agent: RobotSpider
Disallow: /

User-agent: Rytr
Disallow: /

User-agent: SaplingAI
Disallow: /

User-agent: Scalenut
Disallow: /

User-agent: Scraper
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: ScriptBook
Disallow: /

User-agent: SEO Content Machine
Disallow: /

User-agent: SEO Robot
Disallow: /

User-agent: Sentibot
Disallow: /

User-agent: Sidetrade
Disallow: /

User-agent: Simplified AI
Disallow: /

User-agent: Sitefinity
Disallow: /

User-agent: Skydancer
Disallow: /

User-agent: SlickWrite
Disallow: /

User-agent: Sonic
Disallow: /

User-agent: Spin Rewriter
Disallow: /

User-agent: Spinbot
Disallow: /

User-agent: Stability
Disallow: /

User-agent: StableDiffusionBot
Disallow: /

User-agent: Sudowrite
Disallow: /

User-agent: Super Agent
Disallow: /

User-agent: Surfer AI
Disallow: /

User-agent: Text Blaze
Disallow: /

User-agent: TextCortex
Disallow: /

User-agent: The Knowledge AI
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Vidnami AI
Disallow: /

User-agent: Webzio
Disallow: /

User-agent: Whisper
Disallow: /

User-agent: WordAI
Disallow: /

User-agent: Wordtune
Disallow: /

User-agent: WormsGTP
Disallow: /

User-agent: WPBot
Disallow: /

User-agent: Writecream
Disallow: /

User-agent: WriterZen
Disallow: /

User-agent: Writescope
Disallow: /

User-agent: Writesonic
Disallow: /

User-agent: xAI
Disallow: /

User-agent: xBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Zero GTP
Disallow: /

User-agent: Zerochat
Disallow: /

User-agent: Zhipu
Disallow: /

User-agent: Zimm
Disallow: /

# Disallow access to specific sensitive directories
Disallow: /bin/
Disallow: /Content/
Disallow: /LeaveRequestForms/
Disallow: /Properties/
Disallow: /Styles/
Disallow: /Scripts/
Disallow: /Views/
Disallow: /WebReferences/

# Disallow access to specific file types
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.ini$
Disallow: /*.json$
Disallow: /*.yaml$
Disallow: /*.xml$
Disallow: /*.config$