Sitemap: https://www.rivenmaster.org/sitemap.xml

# ============================================================
# Zoekmachines - Toegestaan
# ============================================================

User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

# ============================================================
# AI-bots - Crawlen toegestaan, trainen NIET (zie /ai.txt)
# ============================================================

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot-AI
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: PhindBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: PanguBot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: Sidetrade indexer
Allow: /

User-agent: webz.io
Allow: /

# ============================================================
# Scrapers / dataset-harvesting - Geblokkeerd
# ============================================================

User-agent: AI2Bot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: Scrapy
Disallow: /

# ============================================================
# Generieke HTTP-clients - Geblokkeerd
# ============================================================

User-agent: Apache-HttpClient
Disallow: /

User-agent: curl
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: Java
Disallow: /

User-agent: libwww-perl
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: wget
Disallow: /

# ============================================================
# Alle overige bots - WordPress-specifieke regels
# ============================================================

User-agent: *
Allow: /wp-admin/admin-ajax.php
Allow: /*/*.css
Allow: /*/*.js
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.bmp$
Allow: /*.ico$
Allow: /*.svg$
Allow: /ads.txt
Allow: /app-ads.txt
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: */disclaimer/*
Disallow: *?attachment_id=
Disallow: /*?filtering=
Disallow: *?replytocom=
Disallow: *?replytocom
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /page/
Disallow: /*?p=
Disallow: /*?paged=
Disallow: /*?page=
Disallow: /*?p=*
Disallow: /*&p=*
Disallow: /*?preview=*