# START YOAST BLOCK
# ---------------------------
# Block problem bots completely
# Updated 21 August 2026
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MajesticSEO
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

# AI training crawlers - blocked same as other scrapers
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Google
User-agent: Googlebot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /*?*
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /feed
Disallow: */feed
Disallow: /comments/
Disallow: /trackback/
Allow: /

User-agent: Googlebot-Image
Disallow: /wp-admin/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /*?*
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /feed
Disallow: */feed
Disallow: /comments/
Disallow: /trackback/
Allow: /

User-agent: Googlebot-News
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /*?*
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /feed
Disallow: */feed
Disallow: /comments/
Disallow: /trackback/
Allow: /

# Bing - Moderately permissive
User-agent: Bingbot
Crawl-delay: 1
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /*?*
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /feed
Disallow: */feed
Disallow: /comments/
Disallow: /trackback/
Allow: /

# Social media bots - Allow for link previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Other legitimate search engines
User-agent: DuckDuckBot
Crawl-delay: 2
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /*?*
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /feed
Disallow: */feed
Disallow: /comments/
Disallow: /trackback/
Allow: /

User-agent: Slurp
Crawl-delay: 2
Disallow: /wp-admin/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /*?*
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /feed
Disallow: */feed
Disallow: /comments/
Disallow: /trackback/
Allow: /

# Default for all other bots - restrictive
User-agent: *
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /*?*
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /tag/
Disallow: /category/
Disallow: /author/
Disallow: /feed
Disallow: */feed
Disallow: /comments/
Disallow: /trackback/

Sitemap: https://www.insidekyoto.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK