# Agent authentication: https://mctech.co.za/auth.md
# AI context: https://mctech.co.za/llms.txt
# Full AI context: https://mctech.co.za/llms-full.txt

# ===================================================
# MC Technologies | robots.txt
# Optimized for WordPress, WooCommerce, and Rank Math
# ===================================================

User-agent: *
Content-Signal: ai-train=no, search=yes, ai-input=no
Allow: /

# WordPress admin and system areas
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Block WordPress internal/duplicate areas
Disallow: /cgi-bin/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /comments/
Disallow: /feed/

# Block internal search results
Disallow: /?s=
Disallow: /search/

# Block WooCommerce account/checkout/cart pages
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# Block common filter/sort duplicate URLs
Disallow: /*?orderby=
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?filter_
Disallow: /*?rating_filter=
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?product-page=

# Allow public assets needed for rendering
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# Block common SEO scrapers and harvesters
User-agent: AhrefsBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: SemrushBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: SemrushBot-OCOB
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: MJ12bot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: dotbot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: DataForSeoBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: BLEXBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: PetalBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: AspiegelBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: img2dataset
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: ImagesiftBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: Omgilibot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: Omgili
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: TurnitinBot
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$

User-agent: ByteSpider
Disallow: /
Allow: /auth.md$
Allow: /llms.txt$
Allow: /llms-full.txt$
# XML Sitemap
Sitemap: https://mctech.co.za/sitemap_index.xml

# TikTok product catalog validation
User-agent: TikTokSpider
Allow: /
Allow: /product/
Allow: /wp-content/uploads/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?product-page=