#robots.txt
#EXCEPTIONS
User-agent: *
Allow: /index-gn2.xml
Allow: /MJ12_E8559BE53EE84BCA312D9DD8950DB13E.txt

# ADSENSE
User-agent: Mediapartners-Google 
Disallow:

# ALL AGENTS
User-agent: * 

# disallow all files in these directories
Disallow: /cgi-bin/

# disallow specific pages
Disallow: /login.html
Disallow: /page.html
Disallow: /spip.php
Disallow: /register.html
Disallow: /company/contact-us-ar1064.html
Disallow: /company/privacy-policy-ar1066.html
Disallow: /company/terms-conditions-ar1068.html
Disallow: /company/topspeed-team-ar10684.html
Disallow: /cars/car-news/index2/pictures*.html

# disallow multi-kw systems
Disallow: /searchtags.html
Disallow: /*/searchtags-*.html

# disallow preview pages
Disallow: /preview/*.html

# disallow all files ending with these extensions
Disallow: /*.php
Disallow: /*.inc
#Disallow: /*.js
#Disallow: /*.css
Disallow: /*.txt
Disallow: /*.pdf
#Disallow: /*.xml

# disallow ajax pages
# Disallow /cars/index310/pictures.html

# disallow all files in /admin directorys
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /trackback/
Disallow: /wp-content/plugins/

# disallow to solve bad spydering
# disallow case specific
Disallow: */1006504/
Disallow: /by/authors/$
Disallow: /tellafriend
Disallow: /community
Disallow: /*?var_
Disallow: /*&var_
Disallow: /*?id_
Disallow: /*&id_
Disallow: /*?action=


# disallow all files with? in url
#Disallow: /*?
#Disallow: /*&
Disallow: /*%
Disallow: /*//

# Yandex bot
User-agent: Yandex
Disallow: /

# Ahref - https://ahrefs.com/robot/
User-Agent: AhrefsBot 
disallow: /
User-agent: spbot
Disallow: / 
User-agent: sistrix
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: MJ12bot
Disallow: / 
User-agent: BPImageWalker
Disallow: / 
User-agent: BPImageWalker/2.0
Disallow: / 
User-agent: ia_archiver
Disallow: /
User-agent: archive.org_bot
Disallow: /
User-agent: ia_archiver-web.archive.org 
Disallow: /