# 88Finder robots.txt
# Last updated: 2026-05-13

# Default rules for all bots
User-agent: *
Allow: /

# Explicitly allow important SEO pages
Allow: /search
Allow: /pricing
Allow: /help
Allow: /contact
Allow: /88-days-calculator

# Pillar pages (australian-farm-jobs, verified-farms-australia,
# work-farm-australia are 301'd into /farm-jobs-australia)
Allow: /farm-jobs-australia
Allow: /farm-work-guide

# Farm work type pages
Allow: /fruit-picking-jobs-australia
Allow: /vegetable-picking-australia
Allow: /grape-picking-australia
Allow: /cherry-picking-australia
Allow: /dairy-farm-jobs-australia

# Regional pages
Allow: /farm-work/

# Blog
Allow: /blog/

# Resources
Allow: /resources/

# 88 Days Calculator
Allow: /88-days-calculator

# B2B
Allow: /for-farms

# Disallow technical files (not HTML pages)
Disallow: /manifest.webmanifest
Disallow: /manifest.json
Disallow: /favicon.ico
Disallow: /icon-*.png
Disallow: /apple-touch-icon.png

# Disallow private/admin areas
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /_next/
Disallow: /*.json$
Disallow: /*?*sort=
Disallow: /*?*filter=
# /search is allowed to be crawled; middleware sets X-Robots-Tag: noindex,follow
# so Googlebot can discover farm detail links without indexing search result pages

# Disallow auth pages (now noindex)
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

# Google specific - more aggressive crawling allowed
User-agent: Googlebot
Allow: /

# Bing specific
User-agent: Bingbot
Allow: /

# Block bad bots (keep AhrefsBot/SemrushBot allowed for SEO visibility)
User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: YandexBot
Disallow: /

# Sitemap location
Sitemap: https://88finder.com/sitemap.xml