# https://www.robotstxt.org/robotstxt.html
User-agent: *
# Block authenticated and private pages
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /settings
Disallow: /dashboard
Disallow: /user/
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
# The admin console moved /admin -> /hq and the B2B console lives at /talent;
# both render a 200 SPA shell to a crawler before their guard resolves, so both
# need listing. /admin stays because old links to it are still in the wild.
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
# Single-use ATS setup links (token in the path); never index them.
Disallow: /ats/
# /community and /job-application/ were listed here until 2026-08-13. Both now
# return 410, and a Disallow'd URL cannot be crawled — so Google would never
# see the 410 and would keep the URL indexed. Deindexing requires the crawl.
# API access:
# Block PRIVATE/authenticated API namespaces only. Do NOT blanket-block /api/
# — the client app fetches public read endpoints (jobs, map, analytics) via
# XHR during rendering, and blocking them makes Googlebot render an empty/404
# page. Public JSON is fetchable here but carries an `X-Robots-Tag: noindex`
# response header (see nginx.conf) so it is not indexed as a standalone page.
Disallow: /api/users/
Disallow: /api/saved-jobs
Disallow: /api/saved-searches
Disallow: /api/cover-letter
Disallow: /api/ai/
Disallow: /api/subscription
Disallow: /api/credits
Disallow: /api/referral
Disallow: /api/prerender/
Allow: /api/jobs/
Allow: /api/map/
Allow: /api/analytics/
Allow: /api/og/
# Unlisted internal mockups (share-by-link only)
Disallow: /mockups.html
Disallow: /mockups/
# Unlisted capabilities / sales collateral (share-by-link only)
Disallow: /ayla-capabilities-statement.html
# Allow public content
Allow: /
Allow: /government-jobs
Allow: /jobs
Allow: /jobs/map
Allow: /jobs/category/
Allow: /jobs/state/
Allow: /jobs/type/
Allow: /jobs/role/
# /jobs/salary/ was listed here until 2026-09-06. The route was removed on
# 2026-08-17 (0 impressions, 0 clicks, never crawled) and nginx now returns 410
# for every URL under it — verified with curl on 2026-09-06: /jobs/salary/ and
# /jobs/salary/gs-13 both 410. An Allow: pointing at a gone route invites the
# crawl budget we do have to spend itself on tombstones.
Allow: /jobs/employer/
Allow: /jobs/remote
# /platform, /sourcing, /talent-insights and /government-job-market-data were
# listed here until 2026-09-14; all four are 301s now (see ui/nginx.conf).
# The software and its price list have their own pages again (2026-09-25).
Allow: /software
Allow: /pricing
Allow: /government-hiring-report
# The government hiring data hub; /analytics 301s here since 2026-09-27.
Allow: /government-hiring-data
Allow: /compensation
Allow: /job-alerts
Allow: /about
# /contact stopped being noindex on 2026-08-17: it carries the white-glove
# recruiting form, which is the only place that offer can be reached.
Allow: /contact
Allow: /terms
Allow: /privacy
# The public API documented and the crawler/agent policy (2026-09-15).
Allow: /developers
Allow: /agent-policy
# Dynamic sitemap
Sitemap: https://aylagov.com/sitemap.xml
# AI Crawlers — explicitly allow for discoverability
# See https://aylagov.com/llms.txt for AI-specific information and
# https://aylagov.com/agent-policy for the stated policy in prose. The
# public JSON API is documented at https://aylagov.com/developers; it is
# fetchable but carries X-Robots-Tag: noindex, so it stays out of every
# group's Allow list here. Each vendor's tokens are listed by role, per its
# own documentation: the training crawler, the search-index crawler, and the
# user-initiated fetcher (which may ignore this file anyway).
#
# Two /api/ paths ARE allowed in every group that disallows /api/ (2026-09-22).
# The longest matching rule wins (RFC 9309), so each Allow below outranks the
# shorter `Disallow: /api/`:
# /api/mcp the public MCP server that llms.txt and
# /developers advertise to agents; the Disallow
# was quietly telling those same agents not to
# call it. `Disallow: /api/mcp/` (one character
# longer) keeps the Allow from reaching the
# private, token-gated /api/mcp/recruiting.
# /api/analytics/export/ the Dataset's CSV distribution (per-state open
# roles, employers, median salary, as-of date),
# linked from /analytics and llms.txt. It carries
# no noindex (nginx.conf exempts it).
#
# Robots groups do not combine: a bot that matches its own group ignores
# `User-agent: *` entirely. So every AI group below repeats the private-path
# Disallows from the `*` group (added 2026-09-25; before that GPTBot, ClaudeBot
# and Perplexity were allowed /hq, /talent, /login and the mockups). Keep the
# lists in step when a private path is added.
#
# OpenAI: GPTBot trains, OAI-SearchBot indexes for ChatGPT search,
# ChatGPT-User fetches on a person's request.
User-agent: GPTBot
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: OAI-SearchBot
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: ChatGPT-User
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: Google-Extended
Allow: /
# Anthropic: ClaudeBot trains, Claude-SearchBot indexes for search,
# Claude-User fetches on a person's request. The retired `anthropic-ai` and
# `Claude-Web` tokens were listed here until 2026-09-15; Anthropic's crawler
# documentation no longer names them and no fetch has carried either since
# the July log baseline.
User-agent: ClaudeBot
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: Claude-SearchBot
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: Claude-User
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
# Job pages are closed to five crawlers that send no visitors (2026-11-15):
# PerplexityBot, CCBot, Amazonbot, Bytespider and Meta-ExternalAgent. In the 28
# days to 2026-11-14 nginx's ai_crawl log counted 405K AI-crawler requests, 69%
# of them on job postings, and these five sent almost nobody back:
# PerplexityBot 154,328 crawls 16 Perplexity-User fetches
# Amazonbot 72,505 crawls 0
# Bytespider 33,721 crawls 0
# Meta-ExternalAgent 12,311 crawls 11 Meta user fetches
# CCBot 1,249 crawls 0
# Their groups disallow the job search and every job page (/jobs...), the
# /government-jobs pillar and its series pages, and /job-alerts. Buyer, data,
# report and guide pages stay open to them. Nothing changes for OpenAI's,
# Anthropic's, Google's or Bing's crawlers: ChatGPT sends the job pages'
# visitors (1,527 sessions in those 28 days). Undo: in each of the five groups
# delete the "Job pages" comment and the three Disallow lines under it.
#
# Perplexity: PerplexityBot indexes, Perplexity-User fetches on request.
User-agent: PerplexityBot
# Job pages: closed to this crawler, listed BEFORE Allow: / so a first-match
# parser reads them too (an RFC 9309 parser takes the longest match either way).
Disallow: /jobs
Disallow: /government-jobs
Disallow: /job-alerts
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: Perplexity-User
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: Applebot-Extended
Allow: /
# Amazon: Amazonbot (training and Alexa answers).
User-agent: Amazonbot
# Job pages: closed to this crawler, listed BEFORE Allow: / so a first-match
# parser reads them too (an RFC 9309 parser takes the longest match either way).
Disallow: /jobs
Disallow: /government-jobs
Disallow: /job-alerts
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
# ByteDance: Bytespider (training).
User-agent: Bytespider
# Job pages: closed to this crawler, listed BEFORE Allow: / so a first-match
# parser reads them too (an RFC 9309 parser takes the longest match either way).
Disallow: /jobs
Disallow: /government-jobs
Disallow: /job-alerts
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
# Meta: Meta-ExternalAgent (training and Meta AI). It matched only `*` until 2026-11-15.
User-agent: Meta-ExternalAgent
# Job pages: closed to this crawler, listed BEFORE Allow: / so a first-match
# parser reads them too (an RFC 9309 parser takes the longest match either way).
Disallow: /jobs
Disallow: /government-jobs
Disallow: /job-alerts
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
User-agent: CCBot
# Job pages: closed to this crawler, listed BEFORE Allow: / so a first-match
# parser reads them too (an RFC 9309 parser takes the longest match either way).
Disallow: /jobs
Disallow: /government-jobs
Disallow: /job-alerts
Allow: /
Allow: /api/mcp
Allow: /api/analytics/export/
Disallow: /api/mcp/
Disallow: /api/
Disallow: /dashboard
Disallow: /user/
Disallow: /settings
Disallow: /login
Disallow: /signup
Disallow: /verification-code
Disallow: /logout
Disallow: /resume-builder
Disallow: /cover-letter
Disallow: /oauth/
Disallow: /admin
Disallow: /hq
Disallow: /talent
Disallow: /saved-jobs
Disallow: /my-resumes
Disallow: /ats/
Disallow: /mockups.html
Disallow: /mockups/
Disallow: /ayla-capabilities-statement.html
# Crawl-delay for polite crawling
Crawl-delay: 1