15 lines
288 B
Plaintext
15 lines
288 B
Plaintext
User-agent: *
|
|
Allow: /
|
|
|
|
Sitemap: https://markoveconomics.com/sitemap.xml
|
|
|
|
# Prevent crawling of API endpoints
|
|
Disallow: /api/
|
|
Disallow: /test-
|
|
Disallow: /debug
|
|
|
|
# Prevent crawling of simulation results (if applicable)
|
|
Disallow: /results/
|
|
|
|
# Crawl delay for polite crawling
|
|
Crawl-delay: 10 |