# ---------------------------------------------------------------------------
# ps2onlinegaming.com
# Cloudflare prepends its managed AI-crawler block ABOVE this file, which
# already covers: GPTBot, ClaudeBot, CCBot, Bytespider, Amazonbot,
# Applebot-Extended, Google-Extended, meta-externalagent,
# CloudflareBrowserRenderingCrawler. Everything below is additive.
#
# Deliberately ALLOWED: Googlebot, Bingbot, DuckDuckBot, YandexBot, Applebot.
# ---------------------------------------------------------------------------

# --- SEO / backlink / marketing crawlers -----------------------------------
# No traffic or ranking benefit here; they crawl to resell the data.
# Seen on this board: AhrefsBot 8,059/mo, DotBot 6,747/mo, MJ12bot 1,180/mo,
# SemrushBot (4,045 in one 23h window), Sogou 1,330/mo, Barkrowler 142/mo.
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: rogerbot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: DataForSeoBot
User-agent: serpstatbot
User-agent: SeekportBot
User-agent: MegaIndex
User-agent: SEOkicks
User-agent: spbot
User-agent: LinkpadBot
User-agent: ZoominfoBot
User-agent: Screaming Frog SEO Spider
User-agent: PetalBot
User-agent: Sogou web spider
User-agent: YisouSpider
User-agent: Mediatoolkitbot
User-agent: Buck
Disallow: /

# --- AI / LLM crawlers not in Cloudflare's managed list ---------------------
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Omgili
User-agent: YouBot
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: FriendlyCrawler
User-agent: AI2Bot
User-agent: AI2Bot-Dolma
User-agent: Meta-ExternalFetcher
User-agent: Kangaroo Bot
User-agent: PanguBot
User-agent: Sidetrade indexer bot
User-agent: img2dataset
User-agent: Scrapy
Disallow: /

# --- Generic scripted clients ----------------------------------------------
User-agent: python-requests
User-agent: python-urllib
User-agent: Go-http-client
User-agent: Java
User-agent: libwww-perl
User-agent: httpx
User-agent: node-fetch
User-agent: axios
Disallow: /

# --- Everything else, including the search engines we want ------------------
# These paths do database work and contain nothing worth indexing.
User-agent: *
Disallow: /adm/
Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /cgi-sys/
Disallow: /config/
Disallow: /cron.php
Disallow: /faq.php
Disallow: /files/
Disallow: /includes/
Disallow: /mcp.php
Disallow: /memberlist.php
Disallow: /posting.php
Disallow: /report.php
Disallow: /search.php
Disallow: /store/
Disallow: /ucp.php
Disallow: /viewonline.php
Disallow: /vendor/
Disallow: /app.php/cron/
Disallow: /app.php/help/
Disallow: /app.php/user/
Disallow: /app.php/feed
Disallow: /feed.php

# Session IDs create an infinite crawl space: every link carries a fresh sid,
# so a crawler never runs out of "new" URLs. This is what let the 2026 flood
# run for weeks. Wildcards are supported by Google, Bing and Yandex.
Disallow: /*sid=