User-agent: *
Allow: /

# Allow all search engines to crawl the site
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Disallow crawling of admin or private areas (if any)
Disallow: /admin/
Disallow: /private/
Disallow: /api/internal/

# Allow crawling of important directories
Allow: /public/
Allow: /assets/
Allow: /images/

# Sitemap location
Sitemap: https://gridit.xyz/sitemap.xml

# Crawl delay (optional - helps with server load)
Crawl-delay: 1