Skip to content

Open-source eCommerce platform for C# and ASP.NET Core 10 with MongoDB. Build multi-store shops and multi-vendor marketplaces with B2B/B2C, REST API, plugins, Vue 3 and Docker.

Refactoring - robots.txt #169

KrzysztofPajak97fcc026fd57
diff --git a/src/Business/Grand.Business.Common/Interfaces/Directory/IRobotsTxtService.cs b/src/Business/Grand.Business.Common/Interfaces/Directory/IRobotsTxtService.cs
new file mode 100644
index 000000000..12718d018
--- /dev/null
+++ b/src/Business/Grand.Business.Common/Interfaces/Directory/IRobotsTxtService.cs
@@ -0,0 +1,33 @@
+using Grand.Domain.Common;
+using System.Threading.Tasks;
+
+namespace Grand.Business.Common.Interfaces.Directory
+{
+ public partial interface IRobotsTxtService
+ {
+ /// <summary>
+ /// Get a robots txt
+ /// </summary>
+ /// <param name="storeId">Store identifier</param>
+ /// <returns>RobotsTxt</returns>
+ Task<RobotsTxt> GetRobotsTxt(string storeId = "");
+
+ /// <summary>
+ /// Insert a RobotsTxt
+ /// </summary>
+ /// <param name="robotsTxt">RobotsTxt</param>
+ Task InsertRobotsTxt(RobotsTxt robotsTxt);
+
+ /// <summary>
+ /// Update a RobotsTxt
+ /// </summary>
+ /// <param name="robotsTxt">RobotsTxt</param>
+ Task UpdateRobotsTxt(RobotsTxt robotsTxt);
+
+ /// <summary>
+ /// Delete a RobotsTxt
+ /// </summary>
+ /// <param name="robotsTxt">RobotsTxt</param>
+ Task DeleteRobotsTxt(RobotsTxt robotsTxt);
+ }
+}
diff --git a/src/Business/Grand.Business.Common/Services/Directory/RobotsTxtService.cs b/src/Business/Grand.Business.Common/Services/Directory/RobotsTxtService.cs
new file mode 100644
index 000000000..089b10948
--- /dev/null
+++ b/src/Business/Grand.Business.Common/Services/Directory/RobotsTxtService.cs
@@ -0,0 +1,118 @@
+using Grand.Business.Common.Interfaces.Directory;
+using Grand.Domain.Common;
+using Grand.Domain.Data;
+using Grand.Infrastructure.Caching;
+using Grand.Infrastructure.Caching.Constants;
+using Grand.Infrastructure.Extensions;
+using MediatR;
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace Grand.Business.Common.Services.Directory
+{
+ public partial class RobotsTxtService : IRobotsTxtService
+ {
+
+ #region Fields
+
+ private readonly IRepository<RobotsTxt> _robotsRepository;
+ private readonly IMediator _mediator;
+ private readonly ICacheBase _cacheBase;
+
+ #endregion
+
+ #region Ctor
+
+ /// <summary>
+ /// Ctor
+ /// </summary>
+ /// <param name="robotsRepository">Robots repository</param>
+ /// <param name="mediator">Mediator</param>
+ /// <param name="cacheBase">Cache manager</param>
+ public RobotsTxtService(
+ IRepository<RobotsTxt> robotsRepository,
+ IMediator mediator,
+ ICacheBase cacheBase)
+ {
+ _robotsRepository = robotsRepository;
+ _mediator = mediator;
+ _cacheBase = cacheBase;
+ }
+
+ #endregion
+
+
+ /// <summary>
+ /// get robotsTXT by store ident
+ /// </summary>
+ /// <param name="storeId"></param>
+ /// <returns></returns>
+ public virtual async Task<RobotsTxt> GetRobotsTxt(string storeId = "")
+ {
+ var key = string.Format(CacheKey.ROBOTS_BY_STORE, storeId);
+
+ return await _cacheBase.GetAsync(key, async () =>
+ {
+ var query = from p in _robotsRepository.Table
+ where p.StoreId == storeId
+ select p;
+
+ var robotsTxt = await Task.FromResult(query.FirstOrDefault());
+ return robotsTxt;
+ });
+ }
+
+ /// <summary>
+ /// Insert a robotsTxt
+ /// </summary>
+ /// <param name="robotsTxt">robotsTxt</param>
+ public virtual async Task InsertRobotsTxt(RobotsTxt robotsTxt)
+ {
+ if (robotsTxt == null)
+ throw new ArgumentNullException(nameof(robotsTxt));
+
+ await _robotsRepository.InsertAsync(robotsTxt);
+
+ await _cacheBase.RemoveByPrefix(CacheKey.ROBOTS_ALL_KEY);
+
+ //event notification
+ await _mediator.EntityInserted(robotsTxt);
+ }
+
+ /// <summary>
+ /// Update a robotsTxt
+ /// </summary>
+ /// <param name="robotsTxt">robotsTxt</param>
+ public virtual async Task UpdateRobotsTxt(RobotsTxt robotsTxt)
+ {
+ if (robotsTxt == null)
+ throw new ArgumentNullException(nameof(robotsTxt));
+
+ await _robotsRepository.UpdateAsync(robotsTxt);
+
+ await _cacheBase.RemoveByPrefix(CacheKey.ROBOTS_ALL_KEY);
+
+ //event notification
+ await _mediator.EntityUpdated(robotsTxt);
+ }
+
+ /// <summary>
+ /// Delete a robotsTxt
+ /// </summary>
+ /// <param name="robotsTxt">robotsTxt</param>
+ public virtual async Task DeleteRobotsTxt(RobotsTxt robotsTxt)
+ {
+ if (robotsTxt == null)
+ throw new ArgumentNullException(nameof(robotsTxt));
+
+ await _robotsRepository.DeleteAsync(robotsTxt);
+
+ await _cacheBase.RemoveByPrefix(CacheKey.ROBOTS_ALL_KEY);
+
+ //event notification
+ await _mediator.EntityDeleted(robotsTxt);
+ }
+
+ }
+}
diff --git a/src/Business/Grand.Business.Common/Startup/StartupApplication.cs b/src/Business/Grand.Business.Common/Startup/StartupApplication.cs
index 52747a5be..8ed5d2d91 100644
--- a/src/Business/Grand.Business.Common/Startup/StartupApplication.cs
+++ b/src/Business/Grand.Business.Common/Startup/StartupApplication.cs
@@ -66,6 +66,7 @@ namespace Grand.Business.Common.Startup
serviceCollection.AddScoped<IExchangeRateService, ExchangeRateService>();
serviceCollection.AddScoped<IMeasureService, MeasureService>();
serviceCollection.AddScoped<IGroupService, GroupService>();
+ serviceCollection.AddScoped<IRobotsTxtService, RobotsTxtService>();
}
private void RegisterConfigurationService(IServiceCollection serviceCollection)
{
diff --git a/src/Business/Grand.Business.System/Services/Installation/InstallDataRobotsTxt.cs b/src/Business/Grand.Business.System/Services/Installation/InstallDataRobotsTxt.cs
new file mode 100644
index 000000000..f7db8210f
--- /dev/null
+++ b/src/Business/Grand.Business.System/Services/Installation/InstallDataRobotsTxt.cs
@@ -0,0 +1,62 @@
+using Grand.Business.System.Interfaces.Installation;
+using Grand.Domain.Common;
+using Grand.Domain.Stores;
+using System.Threading.Tasks;
+
+namespace Grand.Business.System.Services.Installation
+{
+ public partial class InstallationService : IInstallationService
+ {
+ protected virtual async Task InstallDataRobotsTxt(
+ Store store)
+ {
+ var url = store.SslEnabled ? store.SecureUrl : store.Url;
+
+ var robotsTxt = new RobotsTxt() {
+ Name = "RobotsTXT",
+ StoreId = store.Id,
+ Text = @$"Sitemap: {url}sitemap.xml
+Host: {url}
+Disallow: /admin
+Disallow: /bin/
+Disallow: /assets/files/
+Disallow: /assets/files/exportimport/
+Disallow: /country/getstatesbycountryid
+Disallow: /install
+Disallow: /setproductreviewhelpfulness
+Disallow: /addproducttocart/catalog/
+Disallow: /addproducttocart/details/
+Disallow: /outofstocksubscriptions/manage
+Disallow: /cart
+Disallow: /changelanguage/*
+Disallow: /checkout/*
+Disallow: /compareproducts
+Disallow: /account/*
+Disallow: /common/customeractioneventurl
+Disallow: /common/getactivepopup
+Disallow: /common/removepopup
+Disallow: /emailwishlist
+Disallow: /login/*
+Disallow: /newsletter/subscriptionactivation
+Disallow: /order/*
+Disallow: /orderdetails
+Disallow: /passwordrecovery/confirm
+Disallow: /popupinteractiveform
+Disallow: /register/*
+Disallow: /merchandisereturn
+Disallow: /merchandisereturn/history
+Disallow: /loyaltypoints/history
+Disallow: /search?
+Disallow: /shoppingcart/*
+Disallow: /storeclosed
+Disallow: /subscribenewsletter/*
+Disallow: /page/authenticate
+Disallow: /uploadfileproductattribute
+Disallow: /uploadfilecheckoutattribute
+Disallow: /wishlist"
+ };
+
+ await _robotsTxtRepository.InsertAsync(robotsTxt);
+ }
+ }
+}
diff --git a/src/Business/Grand.Business.System/Services/Installation/InstallDataStores.cs b/src/Business/Grand.Business.System/Services/Installation/InstallDataStores.cs
index b1eb1475e..699805277 100644
--- a/src/Business/Grand.Business.System/Services/Installation/InstallDataStores.cs
+++ b/src/Business/Grand.Business.System/Services/Installation/InstallDataStores.cs
@@ -12,14 +12,12 @@ namespace Grand.Business.System.Services.Installation
string httpscheme, HostString host,
string companyName, string companyAddress, string companyPhoneNumber, string companyEmail)
{
- var stores = new List<Store>
- {
- new Store
- {
+ var store =
+ new Store {
Name = "Your store name",
Shortcut = "Store",
- Url = $"http://{host}",
- SecureUrl = $"https://{host}",
+ Url = $"http://{host}/",
+ SecureUrl = $"https://{host}/",
SslEnabled = httpscheme.ToLowerInvariant() == "https" ? true : false,
DisplayOrder = 1,
CompanyName = companyName,
@@ -29,11 +27,12 @@ namespace Grand.Business.System.Services.Installation
CompanyVat = null,
CompanyEmail = companyEmail,
CompanyHours = "Monday - Sunday / 8:00AM - 6:00PM",
- Domains = new List<DomainHost>(){ new DomainHost() { HostName = host.Value, Url = $"{httpscheme}://{host}", Primary = true } }
- },
- };
+ Domains = new List<DomainHost>() { new DomainHost() { HostName = host.Value, Url = $"{httpscheme}://{host}", Primary = true } }
+ };
- await _storeRepository.InsertAsync(stores);
+ store = await _storeRepository.InsertAsync(store);
+
+ await InstallDataRobotsTxt(store);
}
}
}
diff --git a/src/Business/Grand.Business.System/Services/Installation/InstallationService.cs b/src/Business/Grand.Business.System/Services/Installation/InstallationService.cs
index 4ac98c4c3..5996a6b1a 100644
--- a/src/Business/Grand.Business.System/Services/Installation/InstallationService.cs
+++ b/src/Business/Grand.Business.System/Services/Installation/InstallationService.cs
@@ -152,6 +152,8 @@ namespace Grand.Business.System.Services.Installation
private readonly IRepository<Banner> _bannerRepository;
private readonly IRepository<Course> _courseRepository;
private readonly IRepository<CourseLevel> _courseLevelRepository;
+ private readonly IRepository<RobotsTxt> _robotsTxtRepository;
+
private readonly IWebHostEnvironment _hostingEnvironment;
private readonly IServiceProvider _serviceProvider;
@@ -267,7 +269,8 @@ namespace Grand.Business.System.Services.Installation
IRepository<InteractiveForm> formRepository,
IRepository<Banner> bannerRepository,
IRepository<Course> courseRepository,
- IRepository<CourseLevel> courseLevelRepository)
+ IRepository<CourseLevel> courseLevelRepository,
+ IRepository<RobotsTxt> robotsTxtRepository)
{
_versionRepository = versionRepository;
@@ -375,7 +378,7 @@ namespace Grand.Business.System.Services.Installation
_bannerRepository = bannerRepository;
_courseRepository = courseRepository;
_courseLevelRepository = courseLevelRepository;
-
+ _robotsTxtRepository = robotsTxtRepository;
_hostingEnvironment = webHostEnvironment;
_serviceProvider = serviceProvider;
}
diff --git a/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateAdminSiteMap.cs b/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateAdminSiteMap.cs
index d36ac6eea..e99389592 100644
--- a/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateAdminSiteMap.cs
+++ b/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateAdminSiteMap.cs
@@ -14,7 +14,7 @@ namespace Grand.Business.System.Services.Migrations._1._1
public int Priority => 0;
public DbVersion Version => new(1, 1);
- public Guid Identity => new("7D8642FC-2FAA-4AC1-B062-5E3513DAC658");
+ public Guid Identity => new("91249F73-CBD2-4007-B2B9-884727279D57");
public string Name => "Update standard admin site map";
/// <summary>
@@ -62,7 +62,11 @@ namespace Grand.Business.System.Services.Migrations._1._1
var childnodeRobots = childnodeDevTools.ChildNodes.FirstOrDefault(x => x.SystemName == "Robot.txt");
if (childnodeRobots != null)
+ {
childnodeRobots.ControllerName = "Maintenance";
+ childnodeRobots.ActionName = "RobotsTxt";
+ childnodeRobots.ResourceName = "Admin.System.RobotsTxt";
+ }
}
var childnodeMaintenance = sitemapSystem.ChildNodes.FirstOrDefault(x => x.SystemName == "Maintenance");
diff --git a/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateResourceString.cs b/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateResourceString.cs
index 68a18537b..807190bab 100644
--- a/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateResourceString.cs
+++ b/src/Business/Grand.Business.System/Services/Migrations/1.1/MigrationUpdateResourceString.cs
@@ -15,7 +15,7 @@ namespace Grand.Business.System.Services.Migrations._1._1
{
public int Priority => 0;
public DbVersion Version => new(1, 1);
- public Guid Identity => new("136CB337-F46C-4228-96C0-2298E2074A90");
+ public Guid Identity => new("E15FFD84-D489-4F6D-BB49-3EEFC7EEE2AC");
public string Name => "Update resource string for english language";
/// <summary>
diff --git a/src/Business/Grand.Business.System/Utilities/StandardAdminSiteMap.cs b/src/Business/Grand.Business.System/Utilities/StandardAdminSiteMap.cs
index 5377023ee..7aaca5077 100644
--- a/src/Business/Grand.Business.System/Utilities/StandardAdminSiteMap.cs
+++ b/src/Business/Grand.Business.System/Utilities/StandardAdminSiteMap.cs
@@ -1056,10 +1056,10 @@ namespace Grand.Business.System.Utilities
},
new AdminSiteMap {
SystemName = "Robot.txt",
- ResourceName = "Admin.System.AdditionsRobotsTxt",
+ ResourceName = "Admin.System.RobotsTxt",
PermissionNames = new List<string> { PermissionSystemName.Maintenance },
ControllerName = "Maintenance",
- ActionName = "AdditionsRobotsTxt",
+ ActionName = "RobotsTxt",
IconClass = "fa fa-dot-circle-o"
}
}
diff --git a/src/Core/Grand.Domain/Common/RobotsTxt.cs b/src/Core/Grand.Domain/Common/RobotsTxt.cs
new file mode 100644
index 000000000..3917d7e10
--- /dev/null
+++ b/src/Core/Grand.Domain/Common/RobotsTxt.cs
@@ -0,0 +1,22 @@
+namespace Grand.Domain.Common
+{
+ public partial class RobotsTxt : BaseEntity
+ {
+
+ /// <summary>
+ /// Gets or sets the name of robots.txt
+ /// </summary>
+ public string Name { get; set; }
+
+ /// <summary>
+ /// Gets or sets the text of robots.txt
+ /// </summary>
+ public string Text { get; set; }
+
+ /// <summary>
+ /// Gets or sets a value store ident
+ /// </summary>
+ public string StoreId { get; set; }
+
+ }
+}
diff --git a/src/Core/Grand.Infrastructure/Caching/Constants/DirectoryCacheKey.cs b/src/Core/Grand.Infrastructure/Caching/Constants/DirectoryCacheKey.cs
index f3fe3d05c..32a0a3831 100644
--- a/src/Core/Grand.Infrastructure/Caching/Constants/DirectoryCacheKey.cs
+++ b/src/Core/Grand.Infrastructure/Caching/Constants/DirectoryCacheKey.cs
@@ -92,6 +92,25 @@
#endregion
+ #region Robots.txt
+
+ /// <summary>
+ /// Key for caching
+ /// </summary>
+ /// <remarks>
+ /// </remarks>
+ public static string ROBOTS_ALL_KEY => "Grand.robots.";
+
+ /// <summary>
+ /// Key for caching by store id
+ /// </summary>
+ /// <remarks>
+ /// {0} : store ID
+ /// </remarks>
+ public static string ROBOTS_BY_STORE => "Grand.robots.id-{0}";
+
+ #endregion
+
#region Country
/// <summary>
/// Key for caching
diff --git a/src/Web/Grand.Web.Admin/Areas/Admin/Views/Maintenance/AdditionsRobotsTxt.cshtml b/src/Web/Grand.Web.Admin/Areas/Admin/Views/Maintenance/AdditionsRobotsTxt.cshtml
deleted file mode 100644
index 1fe2e8cbe..000000000
--- a/src/Web/Grand.Web.Admin/Areas/Admin/Views/Maintenance/AdditionsRobotsTxt.cshtml
+++ /dev/null
@@ -1,70 +0,0 @@
-@model Editor
-@{
- //page title
- ViewBag.Title = Loc["Admin.System.AdditionsRobotsTxt"];
-}
-<link rel="stylesheet" type="text/css" asp-src="/_content/Grand.Web.Admin/administration/codemirror/codemirror.css" />
-<script asp-location="Head" asp-src="/_content/Grand.Web.Admin/administration/codemirror/codemirror.js"></script>
-
-<style>
- .CodeMirror {
- background: #f8f8f8;
- border: 1px solid #eee;
- height: 600px;
- }
-</style>
-<antiforgery-token />
-<div class="row">
- <div class="col-md-12">
- <div class="x_panel light form-fit">
- <div class="x_title">
- <div class="caption level-caption">
- <i class="fa fa-sitemap"></i>
- @Loc["Admin.System.AdditionsRobotsTxt"]
- </div>
- </div>
- <div class="actions">
- <div class="util-btn-margin-bottom-5">
- <button id="btnSave" class="btn btn-success">
- <i class="fa fa fa-check"></i> @Loc["Admin.Common.Save"]
- </button>
- </div>
- </div>
- <div class="x_content form">
-
- <div class="col-12">
- <textarea id="Content" name="Content">@Model.Content</textarea>
- </div>
- <script>
- $(document).ready(function () {
- var editor = CodeMirror.fromTextArea(document.getElementById('Content'), {
- lineNumbers: true
- });
-
- $('#btnSave').click(function () {
- $.ajax({
- cache: false,
- url: "@Url.Action("SaveRobotsTxt", "Maintenance", new { area = Constants.AreaAdmin })",
- data: additionalData(),
- type: 'POST',
- success: function (response) {
- alert(response);
- }
- });
- return false;
- });
-
- function additionalData() {
- var data = {
- content: editor.getValue()
- };
- addAntiForgeryToken(data);
- return data;
- }
-
- });
- </script>
- </div>
- </div>
- </div>
-</div>
\ No newline at end of file
diff --git a/src/Web/Grand.Web.Admin/Areas/Admin/Views/Maintenance/RobotsTxt.cshtml b/src/Web/Grand.Web.Admin/Areas/Admin/Views/Maintenance/RobotsTxt.cshtml
new file mode 100644
index 000000000..9d9434dd3
--- /dev/null
+++ b/src/Web/Grand.Web.Admin/Areas/Admin/Views/Maintenance/RobotsTxt.cshtml
@@ -0,0 +1,68 @@
+@model RobotsTxtModel
+@{
+ //page title
+ ViewBag.Title = Loc["Admin.System.RobotsTxt"];
+}
+<link rel="stylesheet" type="text/css" asp-src="/_content/Grand.Web.Admin/administration/codemirror/codemirror.css" />
+<script asp-location="Head" asp-src="/_content/Grand.Web.Admin/administration/codemirror/codemirror.js"></script>
+
+<style>
+ .CodeMirror {
+ background: #f8f8f8;
+ border: 1px solid #eee;
+ height: 600px;
+ }
+</style>
+
+<form asp-area="@Constants.AreaAdmin" asp-controller="Maintenance" asp-action="RobotsTxt" method="post">
+ <div class="row">
+ <div class="col-md-12">
+ <div class="x_panel light form-fit">
+ <div class="x_title">
+ <div class="caption">
+ <i class="fa fa-fw fa-sliders"></i>
+ @Loc["Admin.System.RobotsTxt"]
+ </div>
+ <div class="actions">
+ <div class="btn-group btn-group-devided">
+ <button class="btn btn-success" type="submit" name="save"><i class="fa fa-check"></i> @Loc["Admin.Common.Save"] </button>
+ </div>
+ </div>
+ <vc:admin-widget widget-zone="customer_user_settings_buttons" additional-data="null" />
+ </div>
+ <div class="x_content">
+ <div class="form-horizontal">
+ <div class="form-body">
+ @await Component.InvokeAsync("StoreScope")
+ </div>
+ </div>
+ </div>
+ <div class="form-horizontal">
+ <div class="form-body">
+ <div class="form-group">
+ <admin-label asp-for="Name" />
+ <div class="col-md-9 col-sm-9">
+ <admin-input asp-for="Name" />
+ <span asp-validation-for="Name"></span>
+ </div>
+ </div>
+ <div class="form-group">
+ <div class="col-12">
+ <textarea id="Text" name="Text">@Model.Text</textarea>
+ <span asp-validation-for="Text"></span>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <script>
+ $(document).ready(function () {
+ CodeMirror.fromTextArea(document.getElementById('Text'), {
+ lineNumbers: true
+ });
+
+ });
+ </script>
+</form>
diff --git a/src/Web/Grand.Web.Admin/Controllers/MaintenanceController.cs b/src/Web/Grand.Web.Admin/Controllers/MaintenanceController.cs
index 9a7e9f76c..572e47b81 100644
--- a/src/Web/Grand.Web.Admin/Controllers/MaintenanceController.cs
+++ b/src/Web/Grand.Web.Admin/Controllers/MaintenanceController.cs
@@ -11,6 +11,7 @@ using Grand.Domain.Media;
using Grand.Domain.Seo;
using Grand.Web.Admin.Extensions;
using Grand.Web.Admin.Models.Common;
+using Grand.Web.Admin.Models.Directory;
using Grand.Web.Common.DataSource;
using Grand.Web.Common.Security.Authorization;
using MediatR;
@@ -33,6 +34,7 @@ namespace Grand.Web.Admin.Controllers
private readonly IDateTimeService _dateTimeService;
private readonly ILanguageService _languageService;
private readonly ITranslationService _translationService;
+ private readonly IRobotsTxtService _robotsTxtService;
private readonly IMediator _mediator;
private readonly IMediaFileStore _mediaFileStore;
@@ -46,6 +48,7 @@ namespace Grand.Web.Admin.Controllers
IDateTimeService dateTimeService,
ILanguageService languageService,
ITranslationService translationService,
+ IRobotsTxtService robotsTxtService,
IMediator mediator,
IMediaFileStore mediaFileStore)
{
@@ -54,6 +57,7 @@ namespace Grand.Web.Admin.Controllers
_dateTimeService = dateTimeService;
_languageService = languageService;
_translationService = translationService;
+ _robotsTxtService = robotsTxtService;
_mediaFileStore = mediaFileStore;
_mediator = mediator;
}
@@ -338,34 +342,50 @@ namespace Grand.Web.Admin.Controllers
return Json(ex.Message);
}
}
- public async Task<IActionResult> AdditionsRobotsTxt()
+
+ public async Task<IActionResult> RobotsTxt()
{
- var model = new Editor();
- var pathFile = _mediaFileStore.Combine("robots.additions.txt");
- var file = await _mediaFileStore.GetFileInfo(pathFile);
- if (file != null)
+ var storeScope = await GetActiveStore();
+
+ var model = new RobotsTxtModel();
+ var robots = await _robotsTxtService.GetRobotsTxt(storeScope);
+ if (robots != null)
{
- model.Content = await _mediaFileStore.ReadAllText(pathFile);
+ model.Name = robots.Name;
+ model.Text = robots.Text;
}
return View(model);
}
-
[HttpPost]
- public async Task<IActionResult> SaveRobotsTxt(string content = "")
+ public async Task<IActionResult> RobotsTxt(RobotsTxtModel model)
{
- try
+ if (ModelState.IsValid)
{
- var pathFile = _mediaFileStore.Combine("robots.additions.txt");
- await _mediaFileStore.WriteAllText(pathFile, content);
+ var storeScope = await GetActiveStore();
- return Json(_translationService.GetResource("Admin.Common.Content.Saved"));
- }
- catch (Exception ex)
- {
- return Json(ex.Message);
+ var robotsTxt = await _robotsTxtService.GetRobotsTxt(storeScope);
+ if (robotsTxt == null)
+ {
+ await _robotsTxtService.InsertRobotsTxt(new Domain.Common.RobotsTxt() {
+ Name = model.Name,
+ Text = model.Text,
+ StoreId = storeScope
+ });
+ }
+ else
+ {
+ robotsTxt.Text = model.Text;
+ robotsTxt.Name = model.Name;
+ await _robotsTxtService.UpdateRobotsTxt(robotsTxt);
+ }
+
+ Success(_translationService.GetResource("Admin.Common.Content.Saved"));
+ return RedirectToAction("RobotsTxt");
}
+
+ return View(model);
}
#endregion
diff --git a/src/Web/Grand.Web.Admin/Models/Directory/RobotsTxtModel.cs b/src/Web/Grand.Web.Admin/Models/Directory/RobotsTxtModel.cs
new file mode 100644
index 000000000..12153e87b
--- /dev/null
+++ b/src/Web/Grand.Web.Admin/Models/Directory/RobotsTxtModel.cs
@@ -0,0 +1,15 @@
+using Grand.Infrastructure.ModelBinding;
+using Grand.Infrastructure.Models;
+
+namespace Grand.Web.Admin.Models.Directory
+{
+ public partial class RobotsTxtModel : BaseEntityModel
+ {
+ [GrandResourceDisplayName("Admin.Configuration.RobotsTxt.Fields.Name")]
+ public string Name { get; set; }
+
+ [GrandResourceDisplayName("Admin.Configuration.RobotsTxt.Fields.Text")]
+ public string Text { get; set; }
+
+ }
+}
\ No newline at end of file
diff --git a/src/Web/Grand.Web.Admin/Validators/Directory/RobotsTxtValidator.cs b/src/Web/Grand.Web.Admin/Validators/Directory/RobotsTxtValidator.cs
new file mode 100644
index 000000000..0f820b98b
--- /dev/null
+++ b/src/Web/Grand.Web.Admin/Validators/Directory/RobotsTxtValidator.cs
@@ -0,0 +1,25 @@
+using FluentValidation;
+using Grand.Business.Common.Interfaces.Localization;
+using Grand.Infrastructure.Validators;
+using Grand.Web.Admin.Models.Directory;
+using System.Collections.Generic;
+
+namespace Grand.Web.Admin.Validators.Directory
+{
+ public class RobotsTxtValidator : BaseGrandValidator<RobotsTxtModel>
+ {
+ public RobotsTxtValidator(
+ IEnumerable<IValidatorConsumer<RobotsTxtModel>> validators,
+ ITranslationService translationService)
+ : base(validators)
+ {
+ RuleFor(x => x.Name)
+ .NotEmpty()
+ .WithMessage(translationService.GetResource("Admin.Configuration.RobotsTxt.Fields.Name.Required"));
+
+ RuleFor(x => x.Text)
+ .NotEmpty()
+ .WithMessage(translationService.GetResource("Admin.Configuration.RobotsTxt.Fields.Text.Required"));
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Web/Grand.Web/App_Data/Resources/DefaultLanguage.xml b/src/Web/Grand.Web/App_Data/Resources/DefaultLanguage.xml
index b562ddc3b..2ee6171d2 100644
--- a/src/Web/Grand.Web/App_Data/Resources/DefaultLanguage.xml
+++ b/src/Web/Grand.Web/App_Data/Resources/DefaultLanguage.xml
@@ -4776,6 +4776,18 @@
<Resource Name="Admin.Configuration.Permissions.Updated">
<Value>The permissions has been updated successfully.</Value>
</Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Name">
+ <Value>Name</Value>
+ </Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Name.Required">
+ <Value>Field name is required</Value>
+ </Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Text">
+ <Value>Content</Value>
+ </Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Text.Required">
+ <Value>Field content is required</Value>
+ </Resource>
<Resource Name="Admin.Configuration.SeNames">
<Value>SEO Names</Value>
</Resource>
@@ -12072,9 +12084,6 @@
<Resource Name="Admin.System">
<Value>System</Value>
</Resource>
- <Resource Name="Admin.System.AdditionsRobotsTxt">
- <Value>Additions Robots.txt</Value>
- </Resource>
<Resource Name="Admin.System.APIUsers">
<Value>Manage api users</Value>
</Resource>
@@ -12423,6 +12432,9 @@
<Resource Name="Admin.System.QueuedEmails.Updated">
<Value>The queued email has been updated successfully.</Value>
</Resource>
+ <Resource Name="Admin.System.RobotsTxt">
+ <Value>Robots.txt</Value>
+ </Resource>
<Resource Name="Admin.System.Roslyn">
<Value>Roslyn compilation</Value>
</Resource>
diff --git a/src/Web/Grand.Web/App_Data/Resources/Upgrade/en_110.xml b/src/Web/Grand.Web/App_Data/Resources/Upgrade/en_110.xml
index f03a736dd..2d6da1bbf 100644
--- a/src/Web/Grand.Web/App_Data/Resources/Upgrade/en_110.xml
+++ b/src/Web/Grand.Web/App_Data/Resources/Upgrade/en_110.xml
@@ -282,4 +282,19 @@
<Resource Name="Account.CustomerMerchandiseReturns.Total">
<Value>Total</Value>
</Resource>
+ <Resource Name="Admin.System.RobotsTxt">
+ <Value>Robots.txt</Value>
+ </Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Name">
+ <Value>Name</Value>
+ </Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Name.Required">
+ <Value>Field name is required</Value>
+ </Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Text">
+ <Value>Content</Value>
+ </Resource>
+ <Resource Name="Admin.Configuration.RobotsTxt.Fields.Text.Required">
+ <Value>Field content is required</Value>
+ </Resource>
</Language>
\ No newline at end of file
diff --git a/src/Web/Grand.Web/Controllers/CommonController.cs b/src/Web/Grand.Web/Controllers/CommonController.cs
index 7e45e2ec8..dce577793 100644
--- a/src/Web/Grand.Web/Controllers/CommonController.cs
+++ b/src/Web/Grand.Web/Controllers/CommonController.cs
@@ -504,7 +504,7 @@ namespace Grand.Web.Controllers
[PublicStore(true)]
public virtual async Task<IActionResult> RobotsTextFile()
{
- var sb = await _mediator.Send(new GetRobotsTextFile());
+ var sb = await _mediator.Send(new GetRobotsTextFile() { StoreId = _workContext.CurrentStore.Id });
return Content(sb, "text/plain");
}
diff --git a/src/Web/Grand.Web/Features/Handlers/Common/GetRobotsTextFileHandler.cs b/src/Web/Grand.Web/Features/Handlers/Common/GetRobotsTextFileHandler.cs
index 5fb65808b..63028a5bf 100644
--- a/src/Web/Grand.Web/Features/Handlers/Common/GetRobotsTextFileHandler.cs
+++ b/src/Web/Grand.Web/Features/Handlers/Common/GetRobotsTextFileHandler.cs
@@ -1,11 +1,6 @@
-using Grand.Business.Common.Interfaces.Localization;
-using Grand.Business.Storage.Interfaces;
-using Grand.Infrastructure;
-using Grand.Infrastructure.Configuration;
+using Grand.Business.Common.Interfaces.Directory;
using Grand.Web.Features.Models.Common;
using MediatR;
-using System.Collections.Generic;
-using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -13,155 +8,22 @@ namespace Grand.Web.Features.Handlers.Common
{
public class GetRobotsTextFileHandler : IRequestHandler<GetRobotsTextFile, string>
{
-
- private readonly IWorkContext _workContext;
- private readonly ILanguageService _languageService;
- private readonly IMediaFileStore _mediaFileStore;
- private readonly AppConfig _config;
+ private readonly IRobotsTxtService _robotsTxtService;
public GetRobotsTextFileHandler(
- IWorkContext workContext,
- ILanguageService languageService,
- IMediaFileStore mediaFileStore,
- AppConfig config)
- {
- _workContext = workContext;
- _languageService = languageService;
- _mediaFileStore = mediaFileStore;
- _config = config;
- }
+ IRobotsTxtService robotsTxtService)
- public async Task<string> Handle(GetRobotsTextFile request, CancellationToken cancellationToken)
{
- return await PrepareRobotsTextFile();
+ _robotsTxtService = robotsTxtService;
}
- private async Task<string> PrepareRobotsTextFile()
+ public async Task<string> Handle(GetRobotsTextFile request, CancellationToken cancellationToken)
{
- var sb = new StringBuilder();
-
- //if robots.txt exists, use it
- var robotsFile = await _mediaFileStore.GetFileInfo("robots.custom.txt");
- if (robotsFile != null)
- {
- //the robots.txt file exists
- string robotsFileContent = await _mediaFileStore.ReadAllText(robotsFile.Name);
- sb.Append(robotsFileContent);
- }
- else
- {
- //doesn't exist. generate
- var disallowPaths = new List<string>
- {
- "/admin",
- "/bin/",
- "/assets/files/",
- "/assets/files/exportimport/",
- "/country/getstatesbycountryid",
- "/install",
- "/setproductreviewhelpfulness",
- };
- var localizableDisallowPaths = new List<string>
- {
- "/addproducttocart/catalog/",
- "/addproducttocart/details/",
- "/outofstocksubscriptions/manage",
- "/cart",
- "/changelanguage/*",
- "/checkout",
- "/checkout/billingaddress",
- "/checkout/completed",
- "/checkout/confirm",
- "/checkout/shippingaddress",
- "/checkout/shippingmethod",
- "/checkout/paymentinfo",
- "/checkout/paymentmethod",
- "/compareproducts",
- "/account/activation",
- "/account/addresses",
- "/account/changepassword",
- "/account/checkusernameavailability",
- "/account/downloadableproducts",
- "/account/info",
- "/account/auctions",
- "/common/customeractioneventurl",
- "/common/getactivepopup",
- "/common/removepopup",
- "/deletepm",
- "/emailwishlist",
- "/login/*",
- "/newsletter/subscriptionactivation",
- "/checkout",
- "/order/history",
- "/orderdetails",
- "/passwordrecovery/confirm",
- "/popupinteractiveform",
- "/register/*",
- "/merchandisereturn",
- "/merchandisereturn/history",
- "/loyaltypoints/history",
- "/search?",
- "/shoppingcart/*",
- "/storeclosed",
- "/subscribenewsletter",
- "/subscribenewsletter/SaveCategories",
- "/page/authenticate",
- "/uploadfileproductattribute",
- "/uploadfilecheckoutattribute",
- "/wishlist",
- };
-
+ var robotsTxt = await _robotsTxtService.GetRobotsTxt(request.StoreId);
+ if (robotsTxt != null)
+ return robotsTxt.Text;
- const string newLine = "\r\n"; //Environment.NewLine
- sb.Append("User-agent: *");
- sb.Append(newLine);
-
- var storeLocation = _workContext.CurrentHost.Url.TrimEnd('/');
-
- //sitemaps
- sb.AppendFormat("Sitemap: {0}/sitemap.xml", storeLocation);
- sb.Append(newLine);
-
- //host
- sb.AppendFormat("Host: {0}", storeLocation);
- sb.Append(newLine);
-
- //usual paths
- foreach (var path in disallowPaths)
- {
- sb.AppendFormat("Disallow: {0}", path);
- sb.Append(newLine);
- }
- //localizable paths (without SEO code)
- foreach (var path in localizableDisallowPaths)
- {
- sb.AppendFormat("Disallow: {0}", path);
- sb.Append(newLine);
- }
- if (_config.SeoFriendlyUrlsForLanguagesEnabled)
- {
- //URLs are localizable. Append SEO code
- foreach (var language in await _languageService.GetAllLanguages(storeId: _workContext.CurrentStore.Id))
- {
- foreach (var path in localizableDisallowPaths)
- {
- sb.AppendFormat("Disallow: /{0}{1}", language.UniqueSeoCode, path);
- sb.Append(newLine);
- }
- }
- }
-
- //load and add robots.txt additions to the end of file.
- var robotsAdditionsFile = await _mediaFileStore.GetFileInfo("robots.additions.txt");
- if (robotsAdditionsFile != null)
- {
- string robotsFileContent = await _mediaFileStore.ReadAllText(robotsAdditionsFile.Name);
- sb.Append(robotsFileContent);
- }
-
- }
- return sb.ToString();
+ return "";
}
-
}
}
diff --git a/src/Web/Grand.Web/Features/Models/Common/GetRobotsTextFile.cs b/src/Web/Grand.Web/Features/Models/Common/GetRobotsTextFile.cs
index 310cc7a26..4a6eed8c9 100644
--- a/src/Web/Grand.Web/Features/Models/Common/GetRobotsTextFile.cs
+++ b/src/Web/Grand.Web/Features/Models/Common/GetRobotsTextFile.cs
@@ -1,12 +1,9 @@
using MediatR;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
namespace Grand.Web.Features.Models.Common
{
- public class GetRobotsTextFile: IRequest<string>
+ public class GetRobotsTextFile : IRequest<string>
{
+ public string StoreId { get; set; }
}
}