# Block Baidu # http://help.baidu.com/question?prod_en=master&class=Baiduspider User-agent: Baiduspider Disallow: / # Block Yandex # https://yandex.com/support/webmaster/controlling-robot/robots-txt.xml User-agent: Yandex Disallow: / # Block Redbot # https://redbot.org/ User-agent: RED Disallow: / # Block Dotbot # https://wowrack.org/ User-agent: dotbot Disallow: / # Block Semrush # https://www.semrush.com/bot/ User-agent: SemrushBot Disallow: / # Block TrovitBot # https://www.trovit.com/bot.html User-agent: trovitBot Disallow: / # Block RogerBot # https://moz.com/help/moz-procedures/crawlers/rogerbot User-agent: rogerbot Disallow: / # Block SeekPort # http://seekport.com/ User-agent: Seekport Disallow: / # Generic bot rules User-agent: * Crawl-delay: 5 Disallow: /backend Disallow: /blog/*/share.html Disallow: /blog/*/share/ Disallow: /blog/*/print.html Disallow: /blog/*/print/ Disallow: /idx/inc/php/ajax/ Disallow: /idx/?refine Disallow: /*/idx/?refine Disallow: /idx/search/ Disallow: /*/idx/search/ Disallow: /idx/search.html Disallow: /*/idx/search.html Disallow: /idx/*/print.html Disallow: /idx/*/map.html Disallow: /idx/*/friend.html Disallow: /idx/*/inquire.html Disallow: /listing/*/inquire/ Disallow: /listing/*/friend/ Disallow: /listing/*/brochure/ Disallow: /listing/*/brochure Disallow: /listing/*/mortgage/ # REWMOD - Added 27/11/23 # Disallow: /listing/*/go-tour/ Disallow: /listing/*/go-tour/* Disallow: /inc/skins/havre-2016/js/widgetbe/agent.php