Here’s a simple way to create a dynamic Sitecore MVC robots.txt.
1) RouteConfig
1 2 3 4 5 6 7 8 9 |
routes.MapRoute( "Robots.txt", "robots.txt", new { controller = "Robots", action = "RobotsText" } ); |
2. Create a RobotsController
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
public class RobotsController : Controller { public FileContentResult RobotsText() { var contentBuilder = new StringBuilder(); contentBuilder.AppendLine("User-agent: *"); if (IsProduction) { contentBuilder.AppendLine("Disallow: /elmah.axd"); contentBuilder.AppendLine("Disallow: /Sitecore"); contentBuilder.AppendLine("Disallow: /sitecore"); contentBuilder.AppendLine("Sitemap: http://www.croda.com/sitemap.xml"); } else { contentBuilder.AppendLine("Disallow: /"); } return File(Encoding.UTF8.GetBytes(contentBuilder.ToString()), "text/plain"); } private bool IsProduction { get { return (ConfigurationManager.AppSettings["IsProduction"] ?? "false") == "true"; } } |
3. Add /robots.txt to IgnoreUrlPrefixes setting in Sitecore.config
4. Ensure <modules runAllManagedModulesForAllRequests=”true”> is set to true within web.config.