# IndianaMat.com robots.txt
# Updated: December 9, 2025

# Default rules for all bots
User-agent: *
Crawl-Delay: 20
Disallow: /*?*sortby=
Disallow: /*?*sortdirection=
Disallow: /*?*advanced_search_submitted=
Disallow: /*?*do=findComment
Disallow: /*?*do=email
Disallow: /*&page=
Disallow: /index.php?/search/
Disallow: /index.php?/profile/*/reputation/
Disallow: /index.php?/profile/*/content/

# Block Facebook's aggressive crawler
User-agent: meta-externalagent
Disallow: /

# Block Baidu (Chinese search - not needed)
User-agent: Baiduspider
Disallow: /

# Block SEMrush (SEO crawler - not needed)
User-agent: SemrushBot
Disallow: /

# Block PetalBot (Huawei search - not needed)
User-agent: PetalBot
Disallow: /

# Block AhrefsBot (SEO crawler)
User-agent: AhrefsBot
Disallow: /

# Block MJ12bot (SEO crawler)
User-agent: MJ12bot
Disallow: /

# Block DotBot
User-agent: DotBot
Disallow: /

# Allow Google but with restrictions
User-agent: Googlebot
Crawl-Delay: 10
Disallow: /*?*sortby=
Disallow: /*?*sortdirection=
Disallow: /*?*advanced_search_submitted=
Disallow: /index.php?/search/
Disallow: /index.php?/profile/*/reputation/

# Allow Bing but with restrictions
User-agent: bingbot
Crawl-Delay: 10
Disallow: /*?*sortby=
Disallow: /*?*sortdirection=
Disallow: /*?*advanced_search_submitted=
Disallow: /index.php?/search/

# Sitemap
Sitemap: https://indianamat.com/sitemap.php