# robots.txt voor Gratis op te halen
# Plaats dit bestand in de hoofdmap van de website:
# https://gratisoptehalen.nl/robots.txt

User-agent: *
Allow: /

# Niet laten indexeren / crawlen
Disallow: /admin/
Disallow: /includes/
Disallow: /cron/
Disallow: /vendor/
Disallow: /logs/
Disallow: /tmp/
Disallow: /backup/
Disallow: /backups/

# Account-, login- en beveiligingspagina's
Disallow: /login
Disallow: /login.php
Disallow: /login_2fa
Disallow: /login_2fa.php
Disallow: /logout
Disallow: /logout.php
Disallow: /register
Disallow: /register.php
Disallow: /register_2fa
Disallow: /register_2fa.php
Disallow: /forgot_password
Disallow: /forgot_password.php
Disallow: /forgot_password_2fa
Disallow: /forgot_password_2fa.php
Disallow: /profile
Disallow: /profile.php
Disallow: /profile_email_verify
Disallow: /profile_email_verify.php
Disallow: /account_export
Disallow: /account_export.php
Disallow: /account_delete
Disallow: /account_delete.php
Disallow: /inbox
Disallow: /inbox.php
Disallow: /thread
Disallow: /thread.php
Disallow: /item_new
Disallow: /item_new.php
Disallow: /item_edit
Disallow: /item_edit.php

# Beheer/monitoring
Disallow: /wie-is-online
Disallow: /wie-is-online.php

# Technische bestanden / upgrades
Disallow: /*.sql$
Disallow: /upgrade_
Disallow: /*upgrade*.sql$
Disallow: /README
Disallow: /PROJECTSTRUCTUUR
Disallow: /SEO_ARTIKEL_URLS
Disallow: /WIE_IS_ONLINE_FIX
Disallow: /COOKIEVRIJE_ASSETS
Disallow: /RECAPTCHA_OPLOSSING

# Zoek/filter parameters liever niet crawlen
Disallow: /*?q=
Disallow: /*?category=
Disallow: /*?place=
Disallow: /*?postcode=
Disallow: /*?status=

# Apple bots blokkeren
User-agent: Applebot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# AI / LLM crawlers blokkeren
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

# Sitemap
Sitemap: https://gratisoptehalen.nl/sitemap.xml