# General instructions for all user agents
User-agent: *
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Disallow: /search
Disallow: /temporary/  # Example for disallowing unnecessary dynamically generated content
Disallow: /private/  # Block access to private content if applicable
Disallow: /blog/wp-admin # wordpress Admin
Disallow: /blog/wp-login.php # protect admin login file

# Sitemap reference
Sitemap: https://alpeshconnect.in/sitemap.xml
Sitemap: https://alpeshconnect.in/imagesitemap.xml


# Block specific web scraping and downloading tools
User-agent: httrack
Disallow: /

User-agent: NetCaptor
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: SpiderKU/0.9
Disallow: /

User-agent: Steeler
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: webcrawler
Disallow: /

User-agent: Web Downloader
Disallow: /

User-agent: WebGather
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Zao
Disallow: /

User-agent: Zeus 2.6
Disallow: /

# Additional Crawling Efficiency Suggestions:
# Consider disallowing common pages that may generate duplicate content or low-value pages:
Disallow: /tag/
Disallow: /category/
Disallow: /archive/
Disallow: /*?*  # Block URLs with query parameters if not necessary for indexing