# robots.txt dynamically created for www.aecdaily.com

User-agent: *
# /images/qr is dynamic images
Disallow: /account_forgot.php
Disallow: /account_register.php
Disallow: /bpl.php
Disallow: /images/qr
Disallow: /pdf.php
Disallow: /requests
Disallow: /templates_c/
Disallow: /templates_dirty/
Disallow: /dberror.php
Disallow: /error.php
Disallow: /generror.php
# Please don't bother indexing our search results, with random queries,
# everything should be linked to something.  We think.
Disallow: /p_search_aec.php

# http://www.80legs.com/webcrawler.html
User-agent: 008
Disallow: /

# https://ahrefs.com/robot/ - I don't hate these guys (yet), but slow 'em down
# At most one every 6 seconds
User-agent: AhrefsBot
Crawl-Delay: 6

# bytedance bytespider tiktok
# there are suggestions that they don't really observe robots.txt
# At most one every 5 seconds
User-agent: Bytespider
Crawl-Delay: 6

# slow them down
# https://www.semrush.com/bot/
User-agent: SemrushBot
Crawl-Delay: 6

# slow them down
# https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots.html?lang=en
User-agent: YandexBot
Crawl-Delay: 6
User-agent: YandexMobileBot
Crawl-Delay: 6

# we have no vegetarian stuff here, get stuffed
User-agent: VegeBot
Disallow: /