# Core: Disallow

User-agent: 008
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: Yandex
Disallow: /

# Core: Allow, Delay

User-Agent: Bingbot
Crawl-Delay: 5

User-Agent: msnbot
Crawl-Delay: 5

# Core: Allow

User-agent: CrownPeakSearchG2Crawler
Allow: /

# Custom

User-agent: *
Allow: /
Disallow: /_master
Disallow: /_site-configuration
Disallow: /_folder-configuration

User-agent: Googlebot
Allow: /
Disallow: /_master
Disallow: /_site-configuration
Disallow: /_folder-configuration

User-agent: bingbot
Allow: /
Disallow: /_master
Disallow: /_site-configuration
Disallow: /_folder-configuration

User-agent: Twitterbot
Allow: /