User-agent: *
 
# WordPress core & admin — never crawl
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /wp-cron.php
Disallow: /wp-json/
 
# WooCommerce — block utility/account/checkout pages (no SEO value, often gated/dynamic)
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-tracking/
Disallow: /wishlist/
Disallow: /credit-card-authorization/
 
# WooCommerce — block faceted filter/sort URLs (thin/duplicate listing views)
# Matches filter_ wherever it appears in the query string, not only right after "?"
Disallow: /*filter_*
Disallow: /*orderby=*
Disallow: /*per_page=*
Disallow: /*shop_view=*
Disallow: /*per_row=*
Disallow: /*add-to-cart=*
Disallow: /*removed_item=*
Disallow: /*undo_item=*
Disallow: /*min_price=*
Disallow: /*max_price=*
 
# WordPress — block internal search, feeds, author/date/tag archives (thin/duplicate content)
Disallow: /?s=
Disallow: /search/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /*/feed/
Disallow: /author/
Disallow: /?author=
Disallow: /tag/
Disallow: /date/
 
# Legacy/duplicate URL patterns
Disallow: /*?replytocom=*
Disallow: /*preview=true*
Disallow: /?p=
Disallow: /?page_id=
 
# Explicitly allow rendering assets rather than blocking whole plugin/theme folders,
# so Google can still fetch CSS/JS needed to render pages correctly
Allow: /wp-content/uploads/
Allow: /*.css
Allow: /*.js
 
# -------------------------------------------------------
# BLOCK KNOWN SCRAPER/AI-TRAINING BOTS
# (kept as separate, specific groups — this is fine since these
#  bots don't also need to match the "*" group's Allow rules)
# -------------------------------------------------------
User-agent: AhrefsBot
Disallow: /
 
User-agent: SemrushBot
Disallow: /
 
User-agent: MJ12bot
Disallow: /
 
User-agent: DotBot
Disallow: /
 
User-agent: BLEXBot
Disallow: /
 
User-agent: PetalBot
Disallow: /
 
User-agent: ia_archiver
Disallow: /
 
User-agent: Bytespider
Disallow: /
 
# -------------------------------------------------------
# SITEMAPS
# -------------------------------------------------------
Sitemap: https://crystaljewelersnc.com/sitemap_index.xml
Sitemap: https://crystaljewelersnc.com/product-sitemap.xml
Sitemap: https://crystaljewelersnc.com/product_cat-sitemap.xml
Sitemap: https://crystaljewelersnc.com/post-sitemap.xml