diff -r 0957fc5e3968 -r 211a3eeacf27 app/soc/content/robots.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/app/soc/content/robots.txt	Mon Aug 18 16:58:23 2008 +0000
@@ -0,0 +1,8 @@
+# Directions for web crawlers.
+# See http://www.robotstxt.org/wc/norobots.html.
+
+User-agent: HTTrack
+User-agent: puf
+User-agent: MSIECrawler
+User-agent: Nutch
+Disallow: /