# mlab.sh - Security Intelligence Platform
# https://mlab.sh

# Everything is crawlable by default, so only Disallow rules below carry any
# weight. There used to be a per-path Allow list here; it was removed because
# `Allow: /` already covers it and the list only created the illusion that a
# page had to be enumerated to be indexed. To keep a page out, add a Disallow.
User-agent: *
Allow: /

# Block auth flows that shouldn't be indexed
Disallow: /auth/reset
Disallow: /auth/forgot
Disallow: /auth/register/organization

# SSO login/consent screens for third-party apps. Nothing here is a landing
# page: /sso/login and /sso/register only make sense mid-flow with a `next`,
# and the rest of /sso/* is OAuth machinery (authorize, token, userinfo).
Disallow: /sso/

# Block utility pages
Disallow: /limited

# /mitre/map?t=... is the same page: `t` is read client-side to preselect
# techniques, the server returns identical HTML either way. No trailing `*`:
# robots.txt matching is prefix based, so this already covers every query
# string, and a literal `*` is a Google/Bing extension a strict parser would
# read as part of the path.
Disallow: /mitre/map?

# Incident Response moved to https://ir.mlab.sh (old /ir/* now 301-redirects there; indexed via ir.mlab.sh's own robots.txt/sitemap)

# TPRM moved to https://tprm.mlab.sh (old /tprm/* now 301-redirects there; indexed via tprm.mlab.sh's own robots.txt/sitemap)

# Help Center + Developer docs moved to https://doc.mlab.sh (old /helpcenter/* and /developer/documentation now 301-redirect there)

# Crawl-delay for polite crawling
Crawl-delay: 2

# Sitemaps
Sitemap: https://mlab.sh/sitemaps/main.xml
Sitemap: https://mlab.sh/sitemaps/blog.xml

# Security policy
# https://mlab.sh/security.txt