User-agent: * Allow: / # Sitemap location Sitemap: https://tecopedia.com/sitemap.xml # Crawl-delay for all bots Crawl-delay: 10 # Disallow specific files/directories if needed # Disallow: /admin/ # Disallow: /private/ # Allow all content for search engines Allow: / Allow: /blog.html Allow: /about.html Allow: /contact.html Allow: /posts/ Allow: /css/ Allow: /js/