# robots.txt for flur.ee
#
# /x/ holds unlisted internal prototypes shared by link only. Only the /x/
# prefix is named here, never the individual page slugs — this file is public,
# so listing a full path would defeat the point. The real no-index enforcement
# is the X-Robots-Tag header set for /x/** in customHttp.yml, plus a <meta
# name="robots" noindex> in each page; those work even if a URL leaks. This
# Disallow is the belt-and-braces layer that also stops non-indexing scrapers
# from crawling the prefix at all.
#
# Named groups below override the "*" group for those agents, so every rule
# has to be repeated in each one.

User-agent: *
Allow: /
Disallow: /dev/
Disallow: /brand-imagery
Disallow: /concepts-dev/
Disallow: /x/

User-agent: Googlebot
Allow: /
Disallow: /dev/
Disallow: /brand-imagery
Disallow: /concepts-dev/
Disallow: /x/

User-agent: Bingbot
Allow: /
Disallow: /dev/
Disallow: /brand-imagery
Disallow: /concepts-dev/
Disallow: /x/

# Link-preview fetchers — blocked from /x/ so pasting an unlisted URL into
# Slack, X, or LinkedIn does not trigger a crawl or generate a public card.
User-agent: Twitterbot
Allow: /
Disallow: /x/

User-agent: facebookexternalhit
Allow: /
Disallow: /x/

Sitemap: https://flur.ee/sitemap.xml