# robots.txt for MrsCrochetWorld # Optimized for Shopify SEO, Google/Bing indexing and AI answer-engine visibility User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /103074890058/checkouts Disallow: /103074890058/orders Disallow: /search Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop # Duplicate / parameter / preview URLs Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* # Shopify collection filter and sorting duplicates Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* # Blog tag duplicate URLs Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* # Legacy WordPress/WooCommerce URLs (site ran on WP before the Shopify migration). # These paths never existed on Shopify — they only produce 404s or duplicate-homepage # crawls and were wasting crawl budget that should go to real product/blog pages. Disallow: /shop-2/ Disallow: /shop/ Disallow: /product-tag/ Disallow: /product-category/ Disallow: /brand/ Disallow: /tag/ Disallow: /category/ Disallow: /cart-2 Disallow: /store-listing Disallow: /mi-cuenta Disallow: /carrito Disallow: /tienda Disallow: /boutique Disallow: /project/ Disallow: /wp-admin Disallow: /wp-login* Disallow: /es/ Disallow: /de/ Disallow: /fr/ Disallow: /*page_id=* # Shopify system URLs Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js # Policy pages: keep blocked if they are standard Shopify policy pages Disallow: /policies/ Disallow: /*/policies/ Sitemap: https://mrscrochetworld.com/sitemap.xml # Google Ads crawler User-agent: AdsBot-Google Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /103074890058/checkouts Disallow: /103074890058/orders Disallow: /search Disallow: /services/login_with_shop Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* # Block low-value crawler User-agent: Nutch Disallow: / # SEO audit crawlers allowed with delay User-agent: AhrefsBot Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop User-agent: MJ12bot Crawl-delay: 10 User-agent: Pinterest Crawl-delay: 1 # AI crawlers allowed for product discovery, citation and answer-engine visibility # Sensitive Shopify/customer/download areas remain blocked. User-agent: GPTBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: ChatGPT-User Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: OAI-SearchBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: PerplexityBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Perplexity-User Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: anthropic-ai Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: ClaudeBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Claude-Web Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Claude-SearchBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Google-Extended Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: CCBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Bytespider Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: meta-externalagent Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Meta-ExternalAgent Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: DuckAssistBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Applebot-Extended Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: Mistral-AI Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: cohere-ai Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / User-agent: YouBot Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /account Disallow: /search Disallow: /services/login_with_shop Allow: / Sitemap: https://mrscrochetworld.com/sitemap.xml