# robots.txt for Colnect Collectors Community - https://colnect.com # # WARNING! Collection of data on Colnect by using automated means is strictly prohibited without written consent from Colnect # # Note you may apply for CAPI (Colnect API) license # # Happy collecting :) # Sitemap: https://colnect.com/s/sitemap_en.xml.gz Sitemap: https://colnect.com/s/sitemap_af.xml.gz Sitemap: https://colnect.com/s/sitemap_sq.xml.gz Sitemap: https://colnect.com/s/sitemap_ar.xml.gz Sitemap: https://colnect.com/s/sitemap_az.xml.gz Sitemap: https://colnect.com/s/sitemap_bn.xml.gz Sitemap: https://colnect.com/s/sitemap_bg.xml.gz Sitemap: https://colnect.com/s/sitemap_be.xml.gz Sitemap: https://colnect.com/s/sitemap_ca.xml.gz Sitemap: https://colnect.com/s/sitemap_hr.xml.gz Sitemap: https://colnect.com/s/sitemap_cs.xml.gz Sitemap: https://colnect.com/s/sitemap_da.xml.gz Sitemap: https://colnect.com/s/sitemap_nl.xml.gz Sitemap: https://colnect.com/s/sitemap_et.xml.gz Sitemap: https://colnect.com/s/sitemap_fi.xml.gz Sitemap: https://colnect.com/s/sitemap_fr.xml.gz Sitemap: https://colnect.com/s/sitemap_fy.xml.gz Sitemap: https://colnect.com/s/sitemap_ka.xml.gz Sitemap: https://colnect.com/s/sitemap_de.xml.gz Sitemap: https://colnect.com/s/sitemap_el.xml.gz Sitemap: https://colnect.com/s/sitemap_he.xml.gz Sitemap: https://colnect.com/s/sitemap_hi.xml.gz Sitemap: https://colnect.com/s/sitemap_hu.xml.gz Sitemap: https://colnect.com/s/sitemap_id.xml.gz Sitemap: https://colnect.com/s/sitemap_it.xml.gz Sitemap: https://colnect.com/s/sitemap_ja.xml.gz Sitemap: https://colnect.com/s/sitemap_ko.xml.gz Sitemap: https://colnect.com/s/sitemap_lv.xml.gz Sitemap: https://colnect.com/s/sitemap_lt.xml.gz Sitemap: https://colnect.com/s/sitemap_ms.xml.gz Sitemap: https://colnect.com/s/sitemap_no.xml.gz Sitemap: https://colnect.com/s/sitemap_fa.xml.gz Sitemap: https://colnect.com/s/sitemap_pl.xml.gz Sitemap: https://colnect.com/s/sitemap_pt.xml.gz Sitemap: https://colnect.com/s/sitemap_ro.xml.gz Sitemap: https://colnect.com/s/sitemap_ru.xml.gz Sitemap: https://colnect.com/s/sitemap_sr.xml.gz Sitemap: https://colnect.com/s/sitemap_si.xml.gz Sitemap: https://colnect.com/s/sitemap_sk.xml.gz Sitemap: https://colnect.com/s/sitemap_sl.xml.gz Sitemap: https://colnect.com/s/sitemap_es.xml.gz Sitemap: https://colnect.com/s/sitemap_sv.xml.gz Sitemap: https://colnect.com/s/sitemap_tl.xml.gz Sitemap: https://colnect.com/s/sitemap_th.xml.gz Sitemap: https://colnect.com/s/sitemap_tr.xml.gz Sitemap: https://colnect.com/s/sitemap_uk.xml.gz Sitemap: https://colnect.com/s/sitemap_ur.xml.gz Sitemap: https://colnect.com/s/sitemap_br.xml.gz Sitemap: https://colnect.com/s/sitemap_zt.xml.gz # msnbot gone berzerk, hopefully temporary, commented out 27/5/2020 #User-agent: msnbot #Disallow: / #User-agent: bingbot #Disallow: / #User-agent: BingPreview #Disallow: / #User-agent: AdIdxBot #Disallow: / # Allow Facebook to preview pages User-agent: facebookexternalhit Disallow: / # Crawlers that are kind enough to obey, but which we'd rather not have # unless they're feeding search engines. User-agent: Baiduspider Disallow: / User-agent: Baiduspider-video Disallow: / User-agent: Baiduspider-image Disallow: / User-agent: UbiCrawler Disallow: / User-agent: BUbiNG Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / User-agent: Arquivo-web-crawler Disallow: / User-agent: BrandVerity/1.0 Disallow: / User-agent: AspiegelBot Disallow: / # Seems legit but for 10 visits a month paying in thousands of daily requests is too much User-agent: DAUM Disallow: / # Some bots are known to be trouble, particularly those designed to copy # entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / User-agent: fast Disallow: / User-agent: MLBot Disallow: / User-agent: dotbot Disallow: / User-agent: 008 Disallow: / User-agent: NextGenSearchBot Disallow: / User-agent: Java/1.6.0_10 Disallow: / User-agent: Spinn3r Disallow: / User-agent: CrazyWebCrawler-Spider Disallow: / User-agent: wget Disallow: / User-agent: grub-client Disallow: / User-agent: k2spider Disallow: / User-agent: NPBot Disallow: / User-agent: WebReaper Disallow: / User-agent: magpie-crawler Disallow: / # Ask Fasterfox plugin not to prefetch User-agent: Fasterfox Disallow: / # Makes a lot of wrong requests - ignores this directive so it's totally blocked User-agent: JikeSpider Disallow: / # Makes a lot of wrong requests User-agent: YYSpider Disallow: / # More useless bots User-agent: TwengaBot Disallow: / User-agent: proximic Disallow: / User-agent: TosCrawler Disallow: / User-agent: grapeshot Disallow: / User-agent: WeSEE Disallow: / User-agent: AhrefsBot disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: BLEXBot Disallow: / User-agent: CamontSpider Disallow: / User-agent: spbot Disallow: / User-agent: CCBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: BDCbot Disallow: / User-agent: MauiBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: VelenPublicWebCrawler Disallow: / # More bad SEO bots - taken from https://www.shoutmeloud.com/what-is-robots-txt-file-and-how-to-optimize-for-wordpress-blogs.html User-agent: Alexibot Disallow: / User-agent: SurveyBot Disallow: / User-agent: Xenu’s Disallow: / User-agent: Xenu’s Link Sleuth 1.1c Disallow: / User-agent: rogerbot Disallow: / User-agent: ia_archiver Disallow: / User-agent: archive.org_bot Disallow: / User-agent: Archive.org Bot Disallow: / User-agent: LinkWalker Disallow: / User-agent: GigaBlast Spider Disallow: / User-agent: ia_archiver-web.archive.org Disallow: / User-agent: PicScout Disallow: / User-agent: TinEye Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: SISTRIX Crawler Disallow: / User-agent: Ezooms Robot Disallow: / User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/) Disallow: / User-agent: WiseGuys Robot Disallow: / User-agent: Turnitin Robot Disallow: / User-agent: Heritrix Disallow: / # Block pricepi User-agent: pimonster Disallow: / User-agent: Pimonster Disallow: / User-agent: Pi-Monster Disallow: / # Block Eniro User-agent: ECCP/1.0 (search@eniro.com) Disallow: / User-agent: Psbot Disallow: / User-agent: YoudaoBot Disallow: / User-agent: NaverBot Disallow: / User-agent: Yeti Disallow: / User-agent: ZBot Disallow: / User-agent: Vagabondo Disallow: / User-agent: LinkWalker Disallow: / User-agent: SimplePie Disallow: / User-agent: Pixray-Seeker Disallow: / User-agent: BoardReader Disallow: / User-agent: Quantify Disallow: / User-agent: Plukkie Disallow: / User-agent: Cuam Disallow: / User-agent: ZoomBot Disallow: / User-Agent: trendictionbot Disallow: / User-agent: AwarioRssBot User-agent: AwarioSmartBot Disallow: / # no traffic from this "search engine" User-agent: SeekportBot Disallow: / # https://megaindex.com/crawler User-agent: MegaIndex.ru Disallow: / User-agent: megaindex.com Disallow: / User-agent: +http://megaindex.com/crawler Disallow: / User-agent: MegaIndex.ru/2.0 Disallow: / User-agent: megaIndex.ru Disallow: / User-agent: Applebot Disallow: / User-agent: Linespider Disallow: / User-agent: hyscore Disallow: / # ias_crawler doesn't seem to obey anything but let's try User-agent: ias_crawler Disallow: / User-agent: DataForSeoBot Disallow: / user-agent: Pinterestbot disallow: / User-agent: CriteoBot/0.1 Disallow: / User-agent: AudigentAdBot Disallow: / User-agent: MaxPointCrawler Disallow: / User-Agent: ZoomBot Disallow: / User-agent: magpie-crawler Disallow: / User-agent: OAI-SearchBot Disallow: / User-agent: barkrowler Disallow: / User-agent: SenutoBot Disallow: / ############################################################################# # BEGINNING AUTOMATED PART - generated 2024-11-19T19:47:30+00:00 User-agent: AdsBot-Google Crawl-delay: 2 Disallow: *.. Disallow: /teletalk/ Disallow: /downloads/ Disallow: /forums/download/ Disallow: /main/set_language/ Disallow: /gu Disallow: /ht Disallow: /kk Disallow: /ml Disallow: /mk Disallow: /pa/ Disallow: /sw Disallow: /ta Disallow: /te Disallow: /*/self$ Disallow: /*/self/ Disallow: /*/new/ Disallow: /*/online/ Disallow: /*/edit/ Disallow: /*/login$ Disallow: /*/account$ Disallow: /*/account/ Disallow: /*/collectors/rate/ Disallow: /*/collectors/log/ Disallow: /*/collectors/list/friends/ Disallow: /*/collectors/list/marked_as_friend/ Disallow: /*/collectors/list/watchlist/ Disallow: /*/collectors/list/best_matches/ Disallow: /*/collectors/log_global/ Disallow: /*/collectors/mark/ Disallow: /*/collectors/my_profile Disallow: /*/collectors/edit_log Disallow: /*/collectors/translation_log Disallow: /*/directory/ Disallow: /*/item/view_collectors/ Disallow: /*/main/ Disallow: /*/collectors/*/active/ Disallow: /*/cart/ Disallow: /*/seller/ Disallow: /*/transaction/ Disallow: /api/ Disallow: /*/api/ Disallow: /capi/ Disallow: /*/capi/ Disallow: /fld/ Disallow: /*/fld/ Disallow: /integrations/ Disallow: /*/integrations/ Disallow: /tool/ Disallow: /*/tool/ Disallow: /*unapproved/ Disallow: /*collection/ Disallow: /*swap_list/ Disallow: /*wish_list/ Disallow: /*ignore/ Disallow: /*buy_list/ Disallow: /*sell_list/ Disallow: /*custom_list Allow: /*by_collection/ Allow: /*by_swap_list/ Allow: /*by_wish_list/ User-agent: Mediapartners-Google Crawl-delay: 5 Disallow: *.. Disallow: /teletalk/ Disallow: /downloads/ Disallow: /forums/download/ Disallow: /main/set_language/ Disallow: /gu Disallow: /ht Disallow: /kk Disallow: /ml Disallow: /mk Disallow: /pa/ Disallow: /sw Disallow: /ta Disallow: /te Disallow: /*/self$ Disallow: /*/self/ Disallow: /*/new/ Disallow: /*/online/ Disallow: /*/edit/ Disallow: /*/login$ Disallow: /*/account$ Disallow: /*/account/ Disallow: /*/collectors/rate/ Disallow: /*/collectors/log/ Disallow: /*/collectors/list/friends/ Disallow: /*/collectors/list/marked_as_friend/ Disallow: /*/collectors/list/watchlist/ Disallow: /*/collectors/list/best_matches/ Disallow: /*/collectors/log_global/ Disallow: /*/collectors/mark/ Disallow: /*/collectors/my_profile Disallow: /*/collectors/edit_log Disallow: /*/collectors/translation_log Disallow: /*/directory/ Disallow: /*/item/view_collectors/ Disallow: /*/main/ Disallow: /*/collectors/*/active/ Disallow: /*/cart/ Disallow: /*/seller/ Disallow: /*/transaction/ Disallow: /api/ Disallow: /*/api/ Disallow: /capi/ Disallow: /*/capi/ Disallow: /fld/ Disallow: /*/fld/ Disallow: /integrations/ Disallow: /*/integrations/ Disallow: /tool/ Disallow: /*/tool/ Disallow: /*unapproved/ Disallow: /*collection/ Disallow: /*swap_list/ Disallow: /*wish_list/ Disallow: /*ignore/ Disallow: /*buy_list/ Disallow: /*sell_list/ Disallow: /*custom_list Allow: /*by_collection/ Allow: /*by_swap_list/ Allow: /*by_wish_list/ User-agent: bingbot Crawl-delay: 2 Disallow: *.. Disallow: /teletalk/ Disallow: /downloads/ Disallow: /forums/download/ Disallow: /main/set_language/ Disallow: /gu Disallow: /ht Disallow: /kk Disallow: /ml Disallow: /mk Disallow: /pa/ Disallow: /sw Disallow: /ta Disallow: /te Disallow: /*/self$ Disallow: /*/self/ Disallow: /*/new/ Disallow: /*/online/ Disallow: /*/edit/ Disallow: /*/login$ Disallow: /*/account$ Disallow: /*/account/ Disallow: /*/collectors/rate/ Disallow: /*/collectors/log/ Disallow: /*/collectors/list/friends/ Disallow: /*/collectors/list/marked_as_friend/ Disallow: /*/collectors/list/watchlist/ Disallow: /*/collectors/list/best_matches/ Disallow: /*/collectors/log_global/ Disallow: /*/collectors/mark/ Disallow: /*/collectors/my_profile Disallow: /*/collectors/edit_log Disallow: /*/collectors/translation_log Disallow: /*/directory/ Disallow: /*/item/view_collectors/ Disallow: /*/main/ Disallow: /*/collectors/*/active/ Disallow: /*/cart/ Disallow: /*/seller/ Disallow: /*/transaction/ Disallow: /api/ Disallow: /*/api/ Disallow: /capi/ Disallow: /*/capi/ Disallow: /fld/ Disallow: /*/fld/ Disallow: /integrations/ Disallow: /*/integrations/ Disallow: /tool/ Disallow: /*/tool/ Disallow: /*unapproved/ Disallow: /*collection/ Disallow: /*swap_list/ Disallow: /*wish_list/ Disallow: /*ignore/ Disallow: /*buy_list/ Disallow: /*sell_list/ Disallow: /*custom_list Allow: /*by_collection/ Allow: /*by_swap_list/ Allow: /*by_wish_list/ User-agent: YandexBot Crawl-delay: 3 Disallow: *.. Disallow: /teletalk/ Disallow: /downloads/ Disallow: /forums/download/ Disallow: /main/set_language/ Disallow: /gu Disallow: /ht Disallow: /kk Disallow: /ml Disallow: /mk Disallow: /pa/ Disallow: /sw Disallow: /ta Disallow: /te Disallow: /*/self$ Disallow: /*/self/ Disallow: /*/new/ Disallow: /*/online/ Disallow: /*/edit/ Disallow: /*/login$ Disallow: /*/account$ Disallow: /*/account/ Disallow: /*/collectors/rate/ Disallow: /*/collectors/log/ Disallow: /*/collectors/list/friends/ Disallow: /*/collectors/list/marked_as_friend/ Disallow: /*/collectors/list/watchlist/ Disallow: /*/collectors/list/best_matches/ Disallow: /*/collectors/log_global/ Disallow: /*/collectors/mark/ Disallow: /*/collectors/my_profile Disallow: /*/collectors/edit_log Disallow: /*/collectors/translation_log Disallow: /*/directory/ Disallow: /*/item/view_collectors/ Disallow: /*/main/ Disallow: /*/collectors/*/active/ Disallow: /*/cart/ Disallow: /*/seller/ Disallow: /*/transaction/ Disallow: /api/ Disallow: /*/api/ Disallow: /capi/ Disallow: /*/capi/ Disallow: /fld/ Disallow: /*/fld/ Disallow: /integrations/ Disallow: /*/integrations/ Disallow: /tool/ Disallow: /*/tool/ Disallow: /*unapproved/ Disallow: /*collection/ Disallow: /*swap_list/ Disallow: /*wish_list/ Disallow: /*ignore/ Disallow: /*buy_list/ Disallow: /*sell_list/ Disallow: /*custom_list Allow: /*by_collection/ Allow: /*by_swap_list/ Allow: /*by_wish_list/ User-agent: SeznamBot Crawl-delay: 800 Disallow: *.. Disallow: /teletalk/ Disallow: /downloads/ Disallow: /forums/download/ Disallow: /main/set_language/ Disallow: /gu Disallow: /ht Disallow: /kk Disallow: /ml Disallow: /mk Disallow: /pa/ Disallow: /sw Disallow: /ta Disallow: /te Disallow: /*/self$ Disallow: /*/self/ Disallow: /*/new/ Disallow: /*/online/ Disallow: /*/edit/ Disallow: /*/login$ Disallow: /*/account$ Disallow: /*/account/ Disallow: /*/collectors/rate/ Disallow: /*/collectors/log/ Disallow: /*/collectors/list/friends/ Disallow: /*/collectors/list/marked_as_friend/ Disallow: /*/collectors/list/watchlist/ Disallow: /*/collectors/list/best_matches/ Disallow: /*/collectors/log_global/ Disallow: /*/collectors/mark/ Disallow: /*/collectors/my_profile Disallow: /*/collectors/edit_log Disallow: /*/collectors/translation_log Disallow: /*/directory/ Disallow: /*/item/view_collectors/ Disallow: /*/main/ Disallow: /*/collectors/*/active/ Disallow: /*/cart/ Disallow: /*/seller/ Disallow: /*/transaction/ Disallow: /api/ Disallow: /*/api/ Disallow: /capi/ Disallow: /*/capi/ Disallow: /fld/ Disallow: /*/fld/ Disallow: /integrations/ Disallow: /*/integrations/ Disallow: /tool/ Disallow: /*/tool/ Disallow: /*unapproved/ Disallow: /*collection/ Disallow: /*swap_list/ Disallow: /*wish_list/ Disallow: /*ignore/ Disallow: /*buy_list/ Disallow: /*sell_list/ Disallow: /*custom_list Allow: /*by_collection/ Allow: /*by_swap_list/ Allow: /*by_wish_list/ User-agent: * Disallow: *.. Disallow: /teletalk/ Disallow: /downloads/ Disallow: /forums/download/ Disallow: /main/set_language/ Disallow: /gu Disallow: /ht Disallow: /kk Disallow: /ml Disallow: /mk Disallow: /pa/ Disallow: /sw Disallow: /ta Disallow: /te Disallow: /*/self$ Disallow: /*/self/ Disallow: /*/new/ Disallow: /*/online/ Disallow: /*/edit/ Disallow: /*/login$ Disallow: /*/account$ Disallow: /*/account/ Disallow: /*/collectors/rate/ Disallow: /*/collectors/log/ Disallow: /*/collectors/list/friends/ Disallow: /*/collectors/list/marked_as_friend/ Disallow: /*/collectors/list/watchlist/ Disallow: /*/collectors/list/best_matches/ Disallow: /*/collectors/log_global/ Disallow: /*/collectors/mark/ Disallow: /*/collectors/my_profile Disallow: /*/collectors/edit_log Disallow: /*/collectors/translation_log Disallow: /*/directory/ Disallow: /*/item/view_collectors/ Disallow: /*/main/ Disallow: /*/collectors/*/active/ Disallow: /*/cart/ Disallow: /*/seller/ Disallow: /*/transaction/ Disallow: /api/ Disallow: /*/api/ Disallow: /capi/ Disallow: /*/capi/ Disallow: /fld/ Disallow: /*/fld/ Disallow: /integrations/ Disallow: /*/integrations/ Disallow: /tool/ Disallow: /*/tool/ Disallow: /*unapproved/ Disallow: /*collection/ Disallow: /*swap_list/ Disallow: /*wish_list/ Disallow: /*ignore/ Disallow: /*buy_list/ Disallow: /*sell_list/ Disallow: /*custom_list Allow: /*by_collection/ Allow: /*by_swap_list/ Allow: /*by_wish_list/ # END AUTOMATED PART - generated 2024-11-19T19:47:30+00:00 #############################################################################