# robots.txt for http://www.mvbuchs.ch/ User-agent: * Allow: / #Allow all User Agents to all Files Disallow: /old_site/ #dont crawling all old sites Disallow: /*!$ #dont crawling sites contains "!" thes are error pages or returns Disallow: /testfolder/ #thes are my test folder dont crawling this