# ---------------------------
# Allow Disallow | by Ompom Oct.26, 2024

# Block all other bots
User-agent: *
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.txt$
Disallow: /*.php$

Sitemap: https://beritaa1.id/sitemap_index.xml
Sitemap: https://beritaa1.id/post-sitemap.xml
Sitemap: https://beritaa1.id/page-sitemap.xml
Sitemap: https://beritaa1.id/category-sitemap.xml
Sitemap: https://beritaa1.id/post_tag-sitemap.xml
Sitemap: https://beritaa1.id/newstopic-sitemap.xml
Sitemap: https://beritaa1.id/author-sitemap.xml

# Allow Googlebot
User-agent: Googlebot
Disallow:
Disallow: /*.php$

# Allow YandexBot
User-agent: Yandex
Disallow:
Disallow: /*.php$

# Allow Bingbot
User-agent: Bingbot
Disallow:
Disallow: /*.php$

# Allow Baiduspider
User-agent: Baiduspider
Disallow:
Disallow: /*.php$

# Allow Facebook scraping bot
User-agent: facebookexternalhit
Disallow:
Disallow: /*.php$

# To block SemrushBot and AI from crawling site for Backlink Audit tool:
User-agent: SemrushBot-BA
Disallow: /
Disallow: /*.php$

User-agent: ChatGPT-User
Disallow: /

User-agent: OpenAI
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: CCBot
Disallow: /

User-Agent: omgili
Disallow: /
 
User-Agent: omgilibot
Disallow: /
 
User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /
 
User-agent: cohere-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: Applebot-Extended
Disallow: /
# ---------------------------