# ====================================================================
# Comprehensive robots.txt for WordPress by inveress.net.au
# ====================================================================


### Sitemap

Sitemap: https://penfieldroses.com.au/sitemap_index.xml


### Allow all crawlers by default.

User-agent: *
Disallow:


# ====================================================================
# SECTION 1: SEARCH ENGINE RULES
# ====================================================================


### Allow major search engines.

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /


### Allow major social media crawlers for link previews.

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /


### Allow major AI crawlers.

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Perplexity
Allow: /

User-agent: Cohere-ai
Allow: /


### Allow major SEO crawlers.

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: dotbot
Allow: /


# ====================================================================
# SECTION 2: WORDPRESS CORE RULES
# ====================================================================


# These rules apply to all crawlers (*).

User-agent: *

# Disallow backend and login pages for security
Disallow: /wp-admin/
Disallow: /wp-login.php

# Allow AJAX functionality, which is needed for some themes and plugins
Allow: /wp-admin/admin-ajax.php

# Disallow core WordPress files that don't need to be crawled
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/

# Allow crawlers to access uploaded files and media
Allow: /wp-content/uploads/

# Disallow unnecessary files
Disallow: /readme.html
Disallow: /license.txt

# Disallow internal search results pages (low value, duplicate content)
Disallow: /search/
Disallow: /*?s=
Disallow: /*/search/

# Disallow author, tag, and other archive pages that can create duplicate content
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: */feed/
Disallow: */trackback/

# Disallow post previews and other dynamic pages
Disallow: /*?p=
Disallow: /*&p=
Disallow: /*&preview=


# ====================================================================
# SECTION 3: BAD BOTS & SCRAPERS
# ====================================================================


User-agent: 008
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: Copier
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: grub
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: libwww
Disallow: /

User-agent: linko
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: RepoMonkey
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: Telesoft
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Xaldon
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: Zeus
Disallow: /