# ==============================================================
# robots.txt — deltaremodels.com
# Delta - Bathroom and Kitchen Remodeling
# Last Updated: June 2026
# Place this file at: https://www.deltaremodels.com/robots.txt
#
# Philosophy: robots.txt is a crawl-budget and indexing tool,
# not a security tool. Malicious bots ignore it entirely, so
# this file focuses on maximizing legitimate search + AI crawler
# access and keeping low-value paths out of crawl budget.
# ==============================================================


# --------------------------------------------------------------
# SECTION 1: Major Search Engine Crawlers — FULL ACCESS
# No crawl-delay: throttling crawlers slows indexing speed,
# which works against ranking goals, not for them.
# --------------------------------------------------------------

User-agent: Googlebot
Allow: /

# Powers Search Console's "Test Live URL" and the Rich Results Test.
# Without its own group it falls through to `User-agent: *` and inherits
# Disallow: /_astro/, making those tools render the site with no CSS or JS.
User-agent: Google-InspectionTool
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /


# --------------------------------------------------------------
# SECTION 2: AI / LLM Crawlers
# These power AI answers and AI search overviews (GEO strategy).
# Fully allowed so Delta appears in AI-generated answers.
# --------------------------------------------------------------

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: ia_archiver
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /


# --------------------------------------------------------------
# SECTION 3: SEO & Analytics Tools — ALLOWED
# No crawl-delay — these tools are how you (and competitors'
# audits referencing you) measure ranking performance.
# --------------------------------------------------------------

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

User-agent: DataForSeoBot
Allow: /


# --------------------------------------------------------------
# SECTION 4: Houzz & Home Improvement Directories — ALLOWED
# --------------------------------------------------------------

User-agent: Houzz
Allow: /

User-agent: PinterestBot
Allow: /


# --------------------------------------------------------------
# SECTION 5: Disallowed Paths — For ALL bots
# Only paths with zero SEO value or that create duplicate/
# low-quality content are blocked here. This protects crawl
# budget; it is not a security control.
# --------------------------------------------------------------

User-agent: *
Allow: /

# Admin & CMS paths
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /dashboard/
Disallow: /login/
Disallow: /logout/
Disallow: /account/
Disallow: /user/

# Staging / Dev environments
Disallow: /staging/
Disallow: /dev/
Disallow: /test/
Disallow: /_dev/
Disallow: /_test/

# Astro / build output internals
Disallow: /_astro/
Disallow: /.astro/
Disallow: /node_modules/
Disallow: /_build/
Disallow: /.git/

# Utility / system paths
Disallow: /cdn-cgi/
Disallow: /.well-known/
Disallow: /tmp/
Disallow: /cache/
Disallow: /logs/

# Form submission / thank-you pages (no search value, avoids thin-content indexing)
Disallow: /thank-you/
Disallow: /form-success/
Disallow: /confirmation/

# Internal search result pages (prevents duplicate/thin content)
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=

# Print pages (duplicate content)
Disallow: /print/
Disallow: /*?print=

# Tracking parameters are deliberately NOT blocked. Each parameterized URL
# carries a self-referencing canonical back to the clean path; blocking them
# here would stop crawlers reading that canonical, so the duplicates would
# never consolidate. Let them be crawled and folded in.


# --------------------------------------------------------------
# SECTION 6: Sitemap Location
# --------------------------------------------------------------

Sitemap: https://www.deltaremodels.com/sitemap-index.xml


# --------------------------------------------------------------
# SECTION 7: Host Directive
# --------------------------------------------------------------

Host: https://www.deltaremodels.com


# ==============================================================
# END OF robots.txt
# Questions? Contact: info@deltaremodels.com
# ==============================================================