User-agent: *
Disallow: /Error/
Disallow: /Static/
Disallow: /Static
Disallow: /Login.aspx
Disallow: /TagService.asmx/GetTagList
Disallow: /BrowseService.asmx/GetTagFriendlyUrl
Disallow: /BrowseService.asmx/GetSupervisors
Disallow: /JsonTagService.ashx
Disallow: /WebResource.axd
Disallow: /OData/V1
Disallow: /Export/
Disallow: /hu/Export/
Disallow: /en/Export/
Disallow: /Search/
Disallow: /hu/Search/
Disallow: /en/Search/
Disallow: /Search
Disallow: /hu/Search
Disallow: /en/Search
Disallow: /Student
Disallow: /hu/Student
Disallow: /en/Student
Disallow: /en/Browse.aspx
Disallow: /hu/Browse.aspx
Disallow: /en/Theses/

Sitemap: http://diplomaterv.vik.bme.hu/Sitemap.xml

# Block the DotBot
User-agent: compatible;+DotBot/1.2;++https://opensiteexplorer.org/dotbot;+help@moz.com
Disallow: /

# Block the Yandex bot (see more: http://help.yandex.com/webmaster/?id=1113851)
User-agent: Yandex
Disallow: /

# Block the PuritySearch.net bot.
User-agent: Purebot 
Disallow: / 

# Block the WBSearchBot (www.warebay.com/bot.html), because it requests invalid URLs.
User-agent: WBSearchBot
Disallow: /

# Block the HTTrack website copier (www.httrack.com), because it downloads the entire site.
User-agent: HTTrack
Disallow: /

# Block the ZmEu bot, because its requests potentially dangerous URLs.
User-agent: ZmEu
Disallow: /

# Block the BLEXBot (http://webmeup.com/crawler.html), because it wants to download the entire site via the search Excel export.
User-agent: BLEXBot
Disallow: /

# Block the SEMrushBot (https://commoncrawl.org/faq/), because it wants to download the entire site via the search Excel export.
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot/6~bl
Disallow:/

# Block the TurnitinBot+(https://turnitin.com/robot/crawlerinfo.html), because it wants to download the entire site via the search Excel export.
User-agent: TurnitinBot
Disallow: /

# Block the CCBot/2.0+(https://commoncrawl.org/faq/), because it wants to download the entire site via the search Excel export.
User-agent: CCBot
Disallow: /
User-agent: CCBot/2.0
Disallow:/

# Block the Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07), because it wants to download the entire site via the search Excel export.
User-agent: Sogou web spider
Disallow:/
User-agent: Sogou web spider/4.0
Disallow: /

# Block the Sogou AhrefsBot/6.1(++http://ahrefs.com/robot/), because it wants to download the ODATA pages.
User-agent: AhrefsBot
Disallow: /
User-agent: AhrefsBot/6.1
Disallow: /

# Block the Sogou MJ12bot/v1.4.8;(+http://mj12bot.com/).
User-agent: MJ12bot
Disallow: /

# Block the MauiBot+(crawler.feedback+dc@gmail.com).
User-agent: MauiBot (crawler.feedback+wc@gmail.com)
Disallow: /

User-agent: PetalBot
Disallow: /

# Mozilla/5.0+AppleWebKit/537.36+(KHTML,+like+Gecko;+compatible;+GPTBot/1.0;++https://openai.com/gptbot)
User-agent: Mozilla/5.0+AppleWebKit/537.36+(KHTML,+like+Gecko;+compatible;+GPTBot/1.0;++https://openai.com/gptbot)
Disallow: /

# Mozilla/5.0+(Linux;+Android+5.0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Mobile+Safari/537.36+(compatible;+Bytespider;+spider-feedback@bytedance.com)
User-agent: Mozilla/5.0+(Linux;+Android+5.0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Mobile+Safari/537.36+(compatible;+Bytespider;+spider-feedback@bytedance.com)
Disallow: /