# robots.txt — infocom.am

# --- Sitemaps ---
Sitemap: https://infocom.am/sitemap_index.xml
Sitemap: https://infocom.am/news-sitemap.xml
Sitemap: https://infocom.am/wp-sitemap.xml

# --- Social preview fetchers ---
User-agent: FacebookExternalHit
Disallow:

User-agent: facebookexternalhit
Disallow:

User-agent: Facebot
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: Pinterestbot
Disallow:

User-agent: Slackbot
Disallow:

User-agent: Slack-ImgProxy
Disallow:

User-agent: Discordbot
Disallow:

User-agent: TelegramBot
Disallow:

User-agent: SkypeUriPreview
Disallow:

User-agent: WhatsApp
Disallow:

# --- Search engines and normal bots ---
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-register.php
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /wptelegram/widget/view/

Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /feed/

# Avoid thin / duplicate / tracking URLs
Disallow: /search
Disallow: /*?s=
Disallow: /*?replytocom=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?share=
Disallow: /*?orderby=
Disallow: /*?filter_
Disallow: /*?amp&

# --- Politeness for aggressive SEO crawlers ---
User-agent: AhrefsBot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

User-agent: MJ12bot
Crawl-delay: 10

# --- Optional: Block AI/model-training crawlers (publisher choice) ---
# Uncomment any you wish to block.
# User-agent: GPTBot
# Disallow: /
# User-agent: OAI-SearchBot
# Disallow: /
# User-agent: ClaudeBot
# Disallow: /
# User-agent: Claude-Web
# Disallow: /
# User-agent: Google-Extended
# Disallow: /
# User-agent: Applebot-Extended
# Disallow: /
# User-agent: meta-externalagent
# Disallow:
# User-agent: CCBot
# Disallow: /
# User-agent: Bytespider
# Disallow: /

# --- Yandex (regional; optional) ---
# Host: infocom.am