#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed

# Gwulo extras
Sitemap: https://gwulo.com/sitemap.xml

# Our serach url is different from the standard one blocked above 
Disallow: /search?

# Block all the other random ? variations like https://gwulo.com/?ref=salesdata.in&page=75
Disallow: /?

# Thumbnail images used in the Random Photo block
Disallow: /sites/default/files/styles/medium/public/

# Pages that are low value for search
Disallow: /media/*/zoom
Disallow: /cart-links/*
Disallow: /user/*/activity
Disallow: /places-missing-current-condition-by-user
Disallow: /modules/contrib/statistics/statistics.php

# Pages that have lists of parameters that means Google sees duplicate pages.
# - block the urls with parameters so Google only sees the base page
Disallow: /node/feed?
# Just index the front page, not any of the following ?page=xxx variants
Disallow: /?page
# Google also tries to index paths like 
# https://gwulo.com/node?page=123&%253B%253B%253B%253B%253B%253B%253B%253Bdestination=
Disallow: /node?page=
# https://gwulo.com/?amp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bamp%3Bdestination=node%253Fpage%3D5&page=61
Disallow: /?amp
# https://gwulo.com/?;;;;;;;;destination=&page=122
Disallow: /?;
# Some requests for the above are submitted with escaped and even double-escaped versions of the ; character
# so include them too
Disallow: /?%3B
Disallow: /?%253B
# https://gwulo.com/media/19091/whatlinkshere?sort=desc&order=Title
Disallow: /media/*/whatlinkshere?
# https://gwulo.com/node/36233/photos?order=name&sort=asc
Disallow: /node/*/photos?
# https://gwulo.com/node/35126/photos-of-person?order=name&sort=desc
Disallow: /node/*/photos-of-person?
# https://gwulo.com/node/53939/whatlinkshere?sort=desc&order=Title
Disallow: /node/*/whatlinkshere?
# https://gwulo.com/photos?decade=All&items_per_page=25&title=&order=name&sort=asc&page=1010
Disallow: /photos?decade=
# https://gwulo.com/photos-by-author/12766?items_per_page=100
Disallow: /photos-by-author/*?
# https://gwulo.com/taxonomy/term/14850/photos?items_per_page=100&order=field_media_sort_string&sort=desc
Disallow: /taxonomy/term/*/photos?
# https://gwulo.com/whats-new?uid=Admin, David&page=969
Disallow: /whats-new?uid=
# https://gwulo.com/whats-new?type[0]=person&page=27
Disallow: /whats-new?type
# https://gwulo.com/forum/2?destination=forum/2?sort%3Dasc%26order=Topic&page=39&sort=asc&order=Last reply
Allow: /forum/2?page=
Disallow: /forum/2?
# https://gwulo.com/list-of-people?title_1=maycock&tid=&items_per_page=25&name=&page=52&order=field_nickname&sort=asc
# Those are old parameters that no longer worked
Allow: /list-of-people?name=
Disallow: /list-of-people?

# Paths from old site that are no longer available
# Temporary - remove these at end of 2022 once the search engines
#   have worked through all the lists of urls to check from the old site
Disallow: /comment/reply_atom/*
Disallow: /sites/gwulo.com/files/styles/
Disallow: /media/*/places
Disallow: /taxonomy/term/*/articles
Disallow: /glossary
Disallow: /taxonomy/term/*/whatlinkshere
Disallow: /taxonomy/term/*/all/feed

# Paths that aren't part of a standard installation
Disallow: /actuator
Disallow: /wp-

# Slow down these crawlers
User-agent: AhrefsBot
Crawl-delay: 20

User-agent: Barkrowler
Crawl-delay: 20

User-agent: BLEXBot
Crawl-delay: 20

User-agent: ClaudeBot
Crawl-delay: 20

User-agent: dotbot
Crawl-delay: 20

User-agent: PetalBot
Crawl-delay: 20

User-agent: SemrushBot
Crawl-delay: 20

# Block these crawlers
User-agent: MegaIndex.ru
Disallow: /

# This one keeps requesting invalid urls, they all have 'https:/wartime-diaries' appended.
User-agent: DataForSeoBot
Disallow: /