# robots.txt file for Time Out .jp
# Updated: 17th March 2026

User-agent: *
# --- TECHNICAL BLOCKLIST ---
# Blocks crawler access to page fragments
Disallow: /*/pages/*/zone/
Disallow: /graffiti/v1/
Disallow: /user/ajax/
Disallow: */pages-ajax
Disallow: */paginate
Disallow: /user/mtoPopupLogin
Disallow: /ajax/mastercard-theme

# --- USER ACCOUNT & AUTHENTICATION ---
Disallow: */user/sign-up
Disallow: */user/myAccount
Disallow: */user/forgotPassword
Disallow: */user/email/review
Disallow: */user/token
Disallow: /user/attending/upcoming*

# --- RSS CLEANUP (Targets legacy folders only) ---
Disallow: */rss/                 # Blocks folders named 'rss'
Disallow: */rss_en_all           # Blocks specific legacy string
Disallow: */rss_ja_all           # Blocks specific legacy string
Disallow: */rss/Features
Disallow: */rss/Events
Disallow: */rss_for_twitter
Disallow: */rss/Venues
# Note: New feeds like /blog/feed.rss remain ALLOWED

# --- SEARCH & DYNAMIC FILTERING ---
Disallow: /search*
Disallow: /*/search*
Disallow: /*/search/
Disallow: /*/search
Disallow: /*/*/search*
Disallow: /*/*/search/
Disallow: /*/*/search
Disallow: /search-listings?*

# --- FILE TYPES & OPERATIONS ---
Disallow: /*/*.php
Disallow: /*/*/bookings

# --- ONLY DISCOVER LOCAL JAPANESE SITEMAPS ---
Sitemap: https://www.timeout.jp/tokyo/ja/sitemap.xml.gz
Sitemap: https://www.timeout.jp/tokyo/ja/news_sitemap.xml
Sitemap: https://www.timeout.jp/time-out-market-osaka/ja/sitemap.xml.gz
Sitemap: https://www.timeout.jp/osaka/ja/sitemap.xml.gz

# --- BOT RESTRICTIONS ---
User-agent: 80bot
Disallow: /

User-agent: genieBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: MLBot
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: OmniExplorer_Bot
Disallow: /

User-agent: spock
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: TurnitinBot
Disallow: /