User-agent: * # list specific files robots are not allowed to index # Disallow:/verify.php Disallow:/Applications/ Disallow:/clients/ Disallow:/newsite/ Disallow:/cgi-bin/ # # End of robots.txt file #