#CoveSys.net #--- # # User-agent: * #Allow: /streamv-erp-help/ # # # # #Disallow: /docs/ Disallow: /videos/ Disallow: /Downloads/ Disallow: /GoToMeeting/ Disallow: /cgi-bin/ Disallow: /PublicDownloads/ Disallow: /Uploads_From_Customers/ Disallow: /emergency/ Disallow: /erpsaas/ Disallow: /mst/ Disallow: /StreamUserSupportForum/ # # # #--- # Start of Blacklisted bots #--- User-agent:libwww Disallow:/ # User-agent:LinkWalker Disallow:/ # User-agent:FindInSiteBot Disallow: /EcommV/ # #--- #End of Blacklisted Bots #--- # # User-agent: Googlebot # Disallow: /marklit # Allow: /marklit/something.pdf # # Some major crawlers support the line 'Allow' # # User-agent: Googlebot # Crawl-delay: 0.5 # # # # Several crawlers support a crawl delay parameter. # The number indicates seconds before the bot can make another request to the server. # # # If a bot is not asking for robots.txt, then you'll need to ban it via .htaccess instead. # # RewriteEngine On # RewriteCond %{HTTP_USER_AGENT} BADBOT # RewriteRule .* - [F] # RewriteEngine On # RewriteCond %{HTTP_USER_AGENT} BADBOT # RewriteRule .* - [F]