# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: *
Disallow: /backend/
Disallow: /plugins/
Disallow: /articles/
Disallow: /admin/
Disallow: /includes/
Disallow: /learning-centre/page/
Disallow: /learning-centre/search
Disallow: /tags/
Disallow: /ms/tags/
Disallow: /widget/
Disallow: /ms/pusat-pembelajaran/laman/
Disallow: /doc/
Disallow: /simple_captcha
Disallow: /subscription/
Disallow: /ms/subscription/
Disallow: /insurance/motorcycle-insurance-road-tax/compare-price
Disallow: /insurance/car-insurance-road-tax/compare-price
Disallow: /insurance/ms/insurans-roadtax-kereta/banding-harga
Disallow: /insurance/ms/insurans-roadtax-motosikal/banding-harga
Disallow: /credit-ca​rd/ambank/ambank-true-by-ambank-vis​a-credit-card
Disallow: /credit-ca​rd/bank-simpanan-nasional/bank-simp​anan-nasional-bsn-1teacherscard-mas​tercard-i
Disallow: /index.htm
Disallow: /learning-​centre/How-to-withdraw-from-EPF-Acc​ount-to-buy-home
Disallow: /ms/kad-kr​edit/standard-chartered/standard-ch​artered-bank-standard-chartered-wor​ldmiles-world-mastercard-credit-car​d
Disallow: /ms/kad-kr%E2%80%8Bedit/aeon-bank
Disallow: /personal-%E2%80%8Bloans
Disallow: /credit-card/*/apply
Disallow: /ms/kad-kredit/*/mohon
Disallow: */verify_otp
Disallow: /insurance/*/sebut-harga
Disallow: /insurance/*/quotation
Disallow: /home-loan/verify_otp
Disallow: /documents/

# allows mortgage urls with query string to be indexed by crawlers
Allow: /home-loan/*

Disallow: /*?

Sitemap: https://www.loanstreet.com.my/en/sitemaps/sitemap.xml.gz
Sitemap: https://www.loanstreet.com.my/ms/sitemaps/sitemap.xml.gz