User-agent: *
Allow: /
# Honeypot probe URLs — not useful for search
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/

# OpenAI GPT crawlers - EXPLICITLY WELCOME
User-agent: GPTBot
Allow: /
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/

Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/

Crawl-delay: 1

User-agent: OAI-SearchBot
Allow: /
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/

Crawl-delay: 1

# Google Gemini crawler
User-agent: Google-Extended
Allow: /
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/

Crawl-delay: 1

# Anthropic Claude crawler
User-agent: anthropic-ai
Allow: /
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/

Crawl-delay: 1

# Meta AI crawler
User-agent: FacebookBot
Allow: /
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/


# Perplexity AI crawler
User-agent: PerplexityBot
Allow: /
Disallow: /salary-at/global-data/export/
Disallow: /compensation/benchmark-2024.json/
Disallow: /api/v1/salary-pulse/
Disallow: /data/city-aggregates.csv/


# Important pages for AI indexing
Allow: /features/
Allow: /about/
Allow: /contact/
Allow: /faq/
Allow: /blog/
Allow: /comparison/
Allow: /how-to-use/
Allow: /privacy-policy/

# Static assets
Allow: /static/
Allow: /favicon.ico

# Sitemaps (same host as sitemap URLs so Bing and Google see the same list)
Sitemap: https://www.in-hand.in/sitemap.xml