# robots.txt for portal-web
# Comprehensive security and crawler management

# Block malicious bots and crawlers
User-agent: BadBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: DotBot
User-agent: BLEXBot
User-agent: YandexBot
User-agent: SeznamBot
User-agent: facebookexternalhit
Disallow: /

# Search engine specific settings
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /.env
Disallow: /config/
Disallow: /logs/
Disallow: /backup/
Disallow: /temp/
Disallow: /cache/
Disallow: /*.bak$
Disallow: /*.tmp$
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.env$
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /.env
Disallow: /config/
Disallow: /logs/
Disallow: /backup/
Disallow: /temp/
Disallow: /cache/
Crawl-delay: 2

User-agent: NaverBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /.env
Disallow: /config/
Disallow: /logs/
Disallow: /backup/
Disallow: /temp/
Disallow: /cache/
Crawl-delay: 1

# Default rules for other crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /.env
Disallow: /config/
Disallow: /logs/
Disallow: /backup/
Disallow: /temp/
Disallow: /cache/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /.git/
Disallow: /.svn/
Disallow: /node_modules/
Disallow: /*.bak$
Disallow: /*.tmp$
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.env$
Disallow: /.*
Crawl-delay: 5

# Sitemap location
Sitemap: https://kochim.com/sitemap.xml

# RSS Feed location
# RSS Feed: https://kochim.com/rss.xml