Skip to content

Open-source eCommerce platform for C# and ASP.NET Core 10 with MongoDB. Build multi-store shops and multi-vendor marketplaces with B2B/B2C, REST API, plugins, Vue 3 and Docker.

1.8 KB
Raw Blame
using Grand.Domain.Common;
using Grand.Domain.Stores;

namespace Grand.Module.Installer.Services;

public partial class InstallationService
{
    protected virtual async Task InstallDataRobotsTxt(
        Store store)
    {
        var url = store.Url;

        var robotsTxt = new RobotsTxt {
            Name = "RobotsTXT",
            StoreId = store.Id,
            //AI crawlers are named on purpose, in the same group as "*": a store wants its
            //products found by assistants too, and naming them states that decision
            //instead of leaving it implied - the rules below apply to them unchanged
            Text = @$"User-agent: *
{RobotsTxtAiCrawlers.UserAgentLines}
Sitemap: {url}sitemap.xml
Host: {url}
Disallow: /admin
Disallow: /bin/
Disallow: /assets/files/
Disallow: /assets/files/exportimport/
Disallow: /country/getstatesbycountryid
Disallow: /install
Disallow: /setproductreviewhelpfulness
Disallow: /addproducttocart/catalog/
Disallow: /addproducttocart/details/
Disallow: /outofstocksubscriptions/manage
Disallow: /cart
Disallow: /changelanguage/*
Disallow: /changecurrency/*
Disallow: /checkout/*
Disallow: /compareproducts
Disallow: /account/*
Disallow: /emailwishlist
Disallow: /login/*
Disallow: /newsletter/subscriptionactivation
Disallow: /order/*
Disallow: /orderdetails
Disallow: /passwordrecovery/confirm
Disallow: /register/*
Disallow: /merchandisereturn
Disallow: /merchandisereturn/history
Disallow: /loyaltypoints/history
Disallow: /search?
Disallow: /shoppingcart/*
Disallow: /storeclosed
Disallow: /subscribenewsletter/*
Disallow: /page/authenticate
Disallow: /uploadfileproductattribute
Disallow: /uploadfilecheckoutattribute
Disallow: /wishlist
Disallow: /quickview/*"
        };

        await _robotsTxtRepository.InsertAsync(robotsTxt);
    }
}