# https://www.robotstxt.org/robotstxt.html
User-agent: *
Disallow: /agent-bio-preview/
Crawl-delay: 10
Sitemap: https://nextsix.com/sitemap.xml
# Block aggressive scrapers
User-agent: scrapy
Disallow: /
User-agent: python-requests
Disallow: /
User-agent: curl
Disallow: /
User-agent: wget
Disallow: /
User-agent: go-http-client
Disallow: /
User-agent: java
Disallow: /
User-agent: okhttp
Disallow: /
User-agent: headless
Disallow: /
User-agent: selenium
Disallow: /
User-agent: webdriver
Disallow: /
User-agent: puppeteer
Disallow: /
User-agent: playwright
Disallow: /
User-agent: Bytespider
Disallow: /