# ============================================ # Agent-Friendly robots.txt for Agentic Commerce # Merchant: State Of Flux # Domain: www.stateofflux.shop # Generated: 2026-02-08 # Powered by Lane (getonlane.com) # ============================================ # This robots.txt is optimized for AI agent accessibility # to enable agentic commerce experiences while protecting # sensitive customer and business data. # # Philosophy: Allow by default, protect only sensitive paths # Compatible with: Shopify, UCP, ACP, Web Bot Auth # ============================================ # ============================================ # SECTION 1: DEFAULT RULES (All Agents) # ============================================ User-agent: * # === PROTECTED PATHS (Security Required) === Disallow: /account Disallow: /account/ Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /25905266766/checkouts Disallow: /25905266766/orders Disallow: /a/downloads/-/* Disallow: /sf_private_access_tokens # === ENCOURAGED: PRODUCT & CONTENT DISCOVERY === Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ # === DUPLICATE CONTENT PREVENTION === Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* # === INTERNAL SHOPIFY PATHS === Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /*?*oseid=* Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /apple-app-site-association Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* # === REMOTE PRODUCT VARIANTS === Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # === STATE OF FLUX-SPECIFIC BLOCKS === Disallow: /pages/search-results-page Disallow: /pages/sca-affiliate-empty-page Disallow: /pages/rewards-page-292083 Disallow: /pages/data-sharing-opt-out Disallow: /search Disallow: /policies/ Disallow: /*/policies/ Disallow: /account/register # ============================================ # SECTION 2: AI SEARCH & ASSISTANT BOTS # ============================================ # --- OpenAI --- User-agent: GPTBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: ChatGPT-User Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: OAI-SearchBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin # --- Anthropic (Claude) --- User-agent: ClaudeBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Claude-User Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Claude-SearchBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: anthropic-ai Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin # --- Perplexity --- User-agent: PerplexityBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Perplexity-User Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin # --- Google AI --- User-agent: Google-Extended Allow: / User-agent: Google-CloudVertexBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Gemini-Deep-Research Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: GoogleAgent-Mariner Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: GoogleAgent-Shopping Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Google-NotebookLM Allow: / # --- Microsoft/Bing --- User-agent: Copilot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: CopilotNative Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: CopilotSapphire Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin # --- Meta AI --- User-agent: meta-externalagent Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: meta-externalfetcher Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin # --- Apple --- User-agent: Applebot-Extended Allow: / # --- Other AI Platforms --- User-agent: DuckAssistBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: MistralAI-User Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: YouBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: cohere-ai Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Amazonbot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: CCBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: DeepSeekBot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Devin Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Diffbot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Timpibot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Thinkbot Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: iaskspider Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: FriendlyCrawler Allow: / Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin # ============================================ # SECTION 3: RATE-LIMITED BOTS # ============================================ User-agent: AhrefsBot Crawl-delay: 10 Allow: /products/ Allow: /collections/ Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: AhrefsSiteAudit Crawl-delay: 10 Allow: /products/ Allow: /collections/ Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: SemrushBot Crawl-delay: 10 Allow: /products/ Allow: /collections/ Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: MJ12bot Crawl-delay: 10 User-agent: Bytespider Crawl-delay: 10 Allow: /products/ Allow: /collections/ Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /admin User-agent: Pinterest Crawl-delay: 1 # ============================================ # SECTION 4: BLOCKED BOTS # ============================================ User-agent: Nutch Disallow: / User-agent: Scrapy Disallow: / User-agent: PetalBot Disallow: / # ============================================ # SECTION 5: GOOGLE ADS BOT # ============================================ User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /25905266766/checkouts Disallow: /25905266766/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js # ============================================ # SECTION 6: SITEMAPS # ============================================ Sitemap: https://www.stateofflux.shop/sitemap.xml # ============================================ # SECTION 7: AGENTIC COMMERCE EXTENSIONS # ============================================ # agents.txt: Coming soon # llms.txt: https://www.stateofflux.shop/llms.txt # Web Bot Auth: In progress # UCP: Ready # ACP: Ready # # Lane (getonlane.com) provides tooling for these standards # ============================================ # # # :::::::: ::::::::::: ::: ::::::::::: :::::::::: # :+: :+: :+: :+: :+: :+: :+: # +:+ +:+ +:+ +:+ +:+ +:+ # +#++:++#++ +#+ +#++:++#++: +#+ +#++:++# # +#+ +#+ +#+ +#+ +#+ +#+ # #+# #+# #+# #+# #+# #+# #+# # ######## ### ### ### ### ########## # # :::::::: :::::::::: # :+: :+: :+: # +:+ +:+ +:+ # +#+ +:+ :#::+::# # +#+ +#+ +#+ # #+# #+# #+# # ######## ### # # :::::::::: ::: ::: ::: ::: ::: # :+: :+: :+: :+: :+: :+: # +:+ +:+ +:+ +:+ +:++:+ # :#::+::# +#+ +#+ +:+ +#++: # +#+ +#+ +#+ +#+ +#+ +#+ # #+# #+# #+# #+# #+# #+# # ### ########## ######## ### ### # # # Welcome to our site, friendly AI agents! # We believe in open commerce for all. # # Powered by Lane (getonlane.com) # Agentic Commerce Infrastructure # # ============================================