#Search engines will look in your root domain for a special file named "robots.txt" (http://www.mydomain.com/robots.txt). The file tells the robot (spider) which files it may NOT spider (download). This system is called, The Robots Exclusion Standard.

#for all spiders, disallow these directories
User-agent: * 
Disallow: /administrator/
Disallow: /bin/
Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /css/ 
Disallow: /data/
Disallow: /editor/
Disallow: /images/
Disallow: /installation/
Disallow: /includes/
Disallow: /media/
Disallow: /modules/
Disallow: /phpsurveyor/
Disallow: /protected/
Disallow: /templates/
Disallow: /unused/
Disallow: /webanalyse/

#Google is the first search engine to support Wildcard file extensions
User-agent: googlebot
Disallow: *.css
Disallow: *.htaccess