User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php #block access to admin section
Disallow: /comments/feed
Disallow: /feed/
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Allow: /wp-admin/admin-ajax.php
Disallow: /page/
Disallow: /2/
Disallow: /3/
Disallow: /4/
Disallow: /5/
Disallow: /6/

User-agent: wget
User-agent: curl
User-agent: MSIECrawler
User-agent: WebCopier
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: libwww
User-agent: muieblackcat
User-agent: duggmirror

User-agent: Slurp
Crawl-delay: 10

Sitemap: https://javiercallejo.net/sitemap_index.xml