# We want to Index all our pages
User-agent: internal_robot
User-aget: AMZNKAssocBot/4.0
Disallow:

# Tells unwanted robots they shouldn't look at any files
#user-agent: aol.com
#user-agent: roadrunner
#user-agent: linkwalker
#user-agent: Pioneer
#user-agent: psbot
user-agent: Googlebot-Image
User-agent: grub-client
user-agent:ia_archiver
user-agent:amzn_assoc
Disallow:  /


# Tells all other robots they can look at any files except these
user-agent: *
Disallow:  /_themes/
Disallow:  /cgi-bin/
Disallow:  /Fan/
Disallow:  /htm/
Disallow:  /images/
Disallow:  /php/
Disallow:  /tmp/
Disallow:  /book.htm
Disallow:  /*.jpg$
Disallow:  /*.JPG$
Disallow:  /*.gif$
Disallow:  /*.GIF$
Disallow:  /*.png$
Disallow:  /*.php$
Disallow:  /*.pl$
Disallow:  /*.js$
Disallow:  /*.htmx$