############################### # # sample robots.txt file for this website # # addresses all robots by using wild card * # User-agent: * # list folders robots are not allowed to index Disallow: /html/images/ Disallow: /html/oldsite/ Disallow: /html/newsite/ Disallow: /html/styles/ Disallow: /oldsite/ # # list specific files robots are not allowed to index # Disallow: /html/apartments_*.html Disallow: /html/apt_*.htm Disallow: /html/apt_*.htm Disallow: /html/footer.htm Disallow: /html/form/sendmail.php # # End of robots.txt file # ###############################