Skip to content

Open-source eCommerce platform for C# and ASP.NET Core 10 with MongoDB. Build multi-store shops and multi-vendor marketplaces with B2B/B2C, REST API, plugins, Vue 3 and Docker.

3.0 KB
Raw Blame
using Grand.Data;
using Grand.Domain.Common;
using Grand.Infrastructure.Caching;
using Grand.Infrastructure.Caching.Constants;
using Grand.Infrastructure.Migrations;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;

namespace Grand.Module.Migration.Migrations._2._4;

/// <summary>
///     Names the AI crawlers in robots.txt, as the installer now does for new stores: they are
///     added as User-agent lines of the existing "User-agent: *" group, so they keep exactly the
///     rules they already followed - nothing is allowed or blocked that was not before.
///     A robots.txt that already names any of them, or has no "*" group, is the operator's own
///     policy and is left untouched.
/// </summary>
public class MigrationRobotsTxtAiCrawlers : IMigration
{
    public int Priority => 1;
    public DbVersion Version => new(2, 4);
    public Guid Identity => new("5B0E3C2A-8D41-4F6E-9A27-C1D84E6F3B95");
    public string Name => "Name the AI crawlers in robots.txt 2.4";

    /// <summary>
    ///     Upgrade process
    /// </summary>
    /// <param name="serviceProvider"></param>
    /// <returns></returns>
    public bool UpgradeProcess(IServiceProvider serviceProvider)
    {
        var logService = serviceProvider.GetRequiredService<ILogger<MigrationRobotsTxtAiCrawlers>>();
        try
        {
            var repository = serviceProvider.GetRequiredService<IRepository<RobotsTxt>>();
            foreach (var robotsTxt in repository.Table.ToList())
            {
                var text = AddAiCrawlers(robotsTxt.Text);
                if (text == robotsTxt.Text) continue;

                robotsTxt.Text = text;
                repository.Update(robotsTxt);
            }

            var cache = serviceProvider.GetService<ICacheBase>();
            cache?.RemoveByPrefix(CacheKey.ROBOTS_ALL_KEY).GetAwaiter().GetResult();
        }
        catch (Exception ex)
        {
            logService.LogError(ex, "UpgradeProcess - RobotsTxtAiCrawlers (2.4)");
        }

        return true;
    }

    public static string AddAiCrawlers(string text)
    {
        if (string.IsNullOrEmpty(text)) return text;

        var lines = text.Split('\n').ToList();
        if (lines.Any(line => RobotsTxtAiCrawlers.UserAgents.Any(agent =>
                IsUserAgentLine(line, agent)))) return text;

        var wildcard = lines.FindIndex(line => IsUserAgentLine(line, "*"));
        if (wildcard < 0) return text;

        var lineEnd = lines[wildcard].EndsWith('\r') ? "\r" : "";
        lines.InsertRange(wildcard + 1, RobotsTxtAiCrawlers.UserAgents.Select(x => "User-agent: " + x + lineEnd));
        return string.Join('\n', lines);
    }

    private static bool IsUserAgentLine(string line, string agent)
    {
        var parts = line.Split(':', 2);
        return parts.Length == 2
               && parts[0].Trim().Equals("User-agent", StringComparison.OrdinalIgnoreCase)
               && parts[1].Trim().Equals(agent, StringComparison.OrdinalIgnoreCase);
    }
}