Apache Solr open-source search software
SOLR-16262: Add a script to add canonical links to Solr 6-8 ref guide pages (#3319)
Houston Putmanad7ceb7ad71e
diff --git a/dev-tools/scripts/refguide/gen-refguide-redirects.py b/dev-tools/scripts/refguide/gen-refguide-redirects.pyindex e7ab1cb1ce0..fac79229702 100755--- a/dev-tools/scripts/refguide/gen-refguide-redirects.py+++ b/dev-tools/scripts/refguide/gen-refguide-redirects.py@@ -129,13 +129,6 @@ def main():print("# Paths we could not map")for key in failed:print("# %s: %s" % (key, failed[key]))-- print("""--# Do not index old reference guide pages on search engines, except for pages that don't exist in 9+-<If "%%{REQUEST_URI} =~ m#/guide/(6|7|8)_.*# && %%{REQUEST_URI} !~ m#/guide/8_11/%s$#">- Header set X-Robots-Tag "noindex,nofollow,noarchive"-</If>""" % old_version_pages_regex)else:out("Regex mappings:")pprint(regex_new)diff --git a/dev-tools/scripts/refguide/htaccess.txt b/dev-tools/scripts/refguide/htaccess.txtindex 419467d09dd..ba9ae2e9f0b 100644--- a/dev-tools/scripts/refguide/htaccess.txt+++ b/dev-tools/scripts/refguide/htaccess.txt@@ -8,6 +8,7 @@ RedirectMatch 301 ^/guide/(major-changes-in-solr-7|major-changes-in-solr-8|solr-# Page renames between 8.x and 9.0RewriteRule ^guide/9_0/solr-tutorial.html /guide/solr/latest/getting-started/solr-tutorial.html [R=301,NE,L]RewriteRule ^guide/a-quick-overview.html /guide/solr/latest/getting-started/introduction.html [R=301,NE,L]+RewriteRule ^guide/a-step-closer.html /guide/solr/latest/configuration-guide/configuration-files.html [R=301,NE,L]RewriteRule ^guide/about-filters.html /guide/solr/latest/indexing-guide/filters.html [R=301,NE,L]RewriteRule ^guide/about-tokenizers.html /guide/solr/latest/indexing-guide/tokenizers.html [R=301,NE,L]RewriteRule ^guide/aws-solrcloud-tutorial.html /guide/solr/latest/getting-started/tutorial-aws.html [R=301,NE,L]@@ -32,6 +33,7 @@ RewriteRule ^guide/files-screen.html /guide/solr/latest/configuration-guide/confRewriteRule ^guide/filter-descriptions.html /guide/solr/latest/indexing-guide/filters.html [R=301,NE,L]RewriteRule ^guide/format-of-solr-xml.html /guide/solr/latest/configuration-guide/configuring-solr-xml.html [R=301,NE,L]RewriteRule ^guide/further-assistance.html https://solr.apache.org/community.html [R=301,NE,L]+RewriteRule ^guide/getting-assistance.html https://solr.apache.org/community.html [R=301,NE,L]RewriteRule ^guide/getting-started-with-solrcloud.html /guide/solr/latest/getting-started/tutorial-solrcloud.html [R=301,NE,L]RewriteRule ^guide/getting-started.html /guide/solr/latest/getting-started/introduction.html [R=301,NE,L]RewriteRule ^guide/how-solrcloud-works.html /guide/solr/latest/deployment-guide/cluster-types.html#solrcloud-mode [R=301,NE,L]@@ -45,10 +47,12 @@ RewriteRule ^guide/introduction-to-scaling-and-distribution.html /guide/solr/latRewriteRule ^guide/introduction-to-solr-indexing.html /guide/solr/latest/getting-started/solr-indexing.html [R=301,NE,L]RewriteRule ^guide/java-properties.html /guide/solr/latest/deployment-guide/jvm-settings.html [R=301,NE,L]RewriteRule ^guide/legacy-scaling-and-distribution.html /guide/solr/latest/deployment-guide/cluster-types.html#user-managed-mode [R=301,NE,L]+RewriteRule ^guide/lib-directives-in-solrconfig.html /guide/solr/latest/configuration-guide/libs.html [R=301,NE,L]RewriteRule ^guide/local-parameters-in-queries.html /guide/solr/latest/query-guide/local-params.html [R=301,NE,L]RewriteRule ^guide/logging.html /guide/solr/latest/deployment-guide/configuring-logging.html [R=301,NE,L]RewriteRule ^guide/major-changes-from-solr-5-to-solr-6.html /guide/solr/latest/upgrade-notes/major-changes-in-solr-6.html [R=301,NE,L]RewriteRule ^guide/making-and-restoring-backups.html /guide/solr/latest/deployment-guide/backup-restore.html [R=301,NE,L]+RewriteRule ^guide/managing-solr.html /guide/solr/latest/deployment-guide/solr-control-script-reference.html [R=301,NE,L]RewriteRule ^guide/merging-indexes.html /guide/solr/latest/configuration-guide/coreadmin-api.html [R=301,NE,L]RewriteRule ^guide/monitoring-solr-with-prometheus-and-grafana.html /guide/solr/latest/deployment-guide/monitoring-with-prometheus-and-grafana.html [R=301,NE,L]RewriteRule ^guide/monitoring-solr.html /guide/solr/latest/deployment-guide/configuring-logging.html [R=301,NE,L]@@ -61,11 +65,14 @@ RewriteRule ^guide/parallel-sql-interface.html /guide/solr/latest/query-guide/sqRewriteRule ^guide/parameter-reference.html /guide/solr/latest/configuration-guide/configuring-solr-xml.html [R=301,NE,L]RewriteRule ^guide/query-settings-in-solrconfig.html /guide/solr/latest/configuration-guide/caches-warming.html [R=301,NE,L]RewriteRule ^guide/query-syntax-and-parsing.html /guide/solr/latest/query-guide/query-syntax-and-parsers.html [R=301,NE,L]+RewriteRule ^guide/read-and-write-side-fault-tolerance.html /guide/solr/latest/deployment-guide/solrcloud-recoveries-and-write-tolerance.html [R=301,NE,L]RewriteRule ^guide/replication-screen.html /guide/solr/latest/deployment-guide/user-managed-index-replication.html [R=301,NE,L]RewriteRule ^guide/requestdispatcher-in-solrconfig.html /guide/solr/latest/configuration-guide/requestdispatcher.html [R=301,NE,L]RewriteRule ^guide/requesthandlers-and-searchcomponents-in-solrconfig.html /guide/solr/latest/configuration-guide/requesthandlers-searchcomponents.html [R=301,NE,L]+RewriteRule ^guide/resource-and-plugin-loading.html /guide/solr/latest/configuration-guide/resource-loading.html [R=301,NE,L]RewriteRule ^guide/running-solr-on-hdfs.html /guide/solr/latest/deployment-guide/solr-on-hdfs.html [R=301,NE,L]RewriteRule ^guide/running-your-analyzer.html /guide/solr/latest/indexing-guide/analysis-screen.html [R=301,NE,L]+RewriteRule ^guide/running-solr.html /guide/solr/latest/deployment-guide/installing-solr.html [R=301,NE,L]RewriteRule ^guide/schema-factory-definition-in-solrconfig.html /guide/solr/latest/configuration-guide/schema-factory.html [R=301,NE,L]RewriteRule ^guide/searching.html /guide/solr/latest/query-guide/query-syntax-and-parsers.html [R=301,NE,L]RewriteRule ^guide/segments-info.html /guide/solr/latest/configuration-guide/index-segments-merging.html [R=301,NE,L]@@ -85,6 +92,11 @@ RewriteRule ^guide/solrcloud-configuration-and-parameters.html /guide/solr/latesRewriteRule ^guide/solrcloud-query-routing-and-read-tolerance.html /guide/solr/latest/deployment-guide/solrcloud-distributed-requests.html [R=301,NE,L]RewriteRule ^guide/solrcloud-resilience.html /guide/solr/latest/deployment-guide/solrcloud-recoveries-and-write-tolerance.html [R=301,NE,L]RewriteRule ^guide/solrcloud.html /guide/solr/latest/deployment-guide/cluster-types.html#solrcloud-mode [R=301,NE,L]+RewriteRule ^guide/statistical-programming.html /guide/solr/latest/query-guide/math-expressions.html [R=301,NE,L]+RewriteRule ^guide/stream-decorators.html /guide/solr/latest/query-guide/stream-decorator-reference.html [R=301,NE,L]+RewriteRule ^guide/stream-evaluators.html /guide/solr/latest/query-guide/stream-evaluator-reference.html [R=301,NE,L]+RewriteRule ^guide/stream-sources.html /guide/solr/latest/query-guide/stream-source-reference.html [R=301,NE,L]+RewriteRule ^guide/time-routed-aliases.html https://solr.apache.org/guide/solr/latest/deployment-guide/aliases.html#time-routed-aliases [R=301,NE,L]RewriteRule ^guide/the-dismax-query-parser.html /guide/solr/latest/query-guide/dismax-query-parser.html [R=301,NE,L]RewriteRule ^guide/the-extended-dismax-query-parser.html /guide/solr/latest/query-guide/edismax-query-parser.html [R=301,NE,L]RewriteRule ^guide/the-query-elevation-component.html /guide/solr/latest/query-guide/query-elevation-component.html [R=301,NE,L]@@ -95,9 +107,11 @@ RewriteRule ^guide/the-term-vector-component.html /guide/solr/latest/query-guideRewriteRule ^guide/the-terms-component.html /guide/solr/latest/query-guide/terms-component.html [R=301,NE,L]RewriteRule ^guide/the-well-configured-solr-instance.html /guide/solr/latest/configuration-guide/configuration-files.html [R=301,NE,L]RewriteRule ^guide/transforming-result-documents.html /guide/solr/latest/query-guide/document-transformers.html [R=301,NE,L]+RewriteRule ^guide/uima-integration.html https://solr.apache.org/guide/7_4/uima-integration.html [R=301,NE,L]RewriteRule ^guide/understanding-analyzers-tokenizers-and-filters.html /guide/solr/latest/indexing-guide/document-analysis.html [R=301,NE,L]RewriteRule ^guide/updatehandlers-in-solrconfig.html /guide/solr/latest/configuration-guide/commits-transaction-logs.html [R=301,NE,L]RewriteRule ^guide/updating-parts-of-documents.html /guide/solr/latest/indexing-guide/partial-document-updates.html [R=301,NE,L]+RewriteRule ^guide/upgrading-solr.html /guide/solr/latest/deployment-guide/upgrading-a-solr-cluster.html [R=301,NE,L]RewriteRule ^guide/uploading-data-with-index-handlers.html /guide/solr/latest/indexing-guide/indexing-with-update-handlers.html [R=301,NE,L]RewriteRule ^guide/uploading-data-with-solr-cell-using-apache-tika.html /guide/solr/latest/indexing-guide/indexing-with-tika.html [R=301,NE,L]RewriteRule ^guide/using-javascript.html /guide/solr/latest/deployment-guide/javascript.html [R=301,NE,L]@@ -107,6 +121,7 @@ RewriteRule ^guide/using-solr-from-ruby.html /guide/solr/latest/deployment-guideRewriteRule ^guide/using-solrj.html /guide/solr/latest/deployment-guide/solrj.html [R=301,NE,L]RewriteRule ^guide/using-the-solr-administration-user-interface.html /guide/solr/latest/getting-started/solr-admin-ui.html [R=301,NE,L]RewriteRule ^guide/using-zookeeper-to-manage-configuration-files.html /guide/solr/latest/deployment-guide/zookeeper-file-management.html [R=301,NE,L]+RewriteRule ^guide/vectorization.html /guide/solr/latest/query-guide/math-expressions.html [R=301,NE,L]RewriteRule ^guide/working-with-currencies-and-exchange-rates.html /guide/solr/latest/indexing-guide/currencies-exchange-rates.html [R=301,NE,L]RewriteRule ^guide/working-with-dates.html /guide/solr/latest/indexing-guide/date-formatting-math.html [R=301,NE,L]RewriteRule ^guide/working-with-enum-fields.html /guide/solr/latest/indexing-guide/enum-fields.html [R=301,NE,L]@@ -118,9 +133,3 @@ RewriteRule ^guide/real-time-get.html /guide/solr/latest/configuration-guide/rea# Removed pages redirected to latest 8.x guideRedirectMatch 301 ^/guide/(adding-custom-plugins-in-solrcloud-mode|blob-store-api|blockjoin-faceting|cdcr-api|cdcr-architecture|cdcr-config|cdcr-operations|colocating-collections|cross-data-center-replication-cdcr|dataimport-screen|errata|metrics-history|migrate-to-policy-rule|putting-the-pieces-together|rule-based-replica-placement|solrcloud-autoscaling-api|solrcloud-autoscaling-auto-add-replicas|solrcloud-autoscaling-fault-tolerance|solrcloud-autoscaling-listeners|solrcloud-autoscaling-overview|solrcloud-autoscaling-policy-preferences|solrcloud-autoscaling-trigger-actions|solrcloud-autoscaling-triggers|solrcloud-autoscaling|suggestions-screen|uploading-structured-data-store-data-with-the-data-import-handler|velocity-response-writer|velocity-search-ui)\.html /guide/8_11/$1.html# Paths we could not map---# Do not index old reference guide pages on search engines, except for pages that don't exist in 9+-<If "%{REQUEST_URI} =~ m#/guide/(6|7|8)_.*# && %{REQUEST_URI} !~ m#/guide/8_11/(adding-custom-plugins-in-solrcloud-mode|blob-store-api|blockjoin-faceting|cdcr-api|cdcr-architecture|cdcr-config|cdcr-operations|colocating-collections|cross-data-center-replication-cdcr|dataimport-screen|errata|metrics-history|migrate-to-policy-rule|putting-the-pieces-together|rule-based-replica-placement|solrcloud-autoscaling-api|solrcloud-autoscaling-auto-add-replicas|solrcloud-autoscaling-fault-tolerance|solrcloud-autoscaling-listeners|solrcloud-autoscaling-overview|solrcloud-autoscaling-policy-preferences|solrcloud-autoscaling-trigger-actions|solrcloud-autoscaling-triggers|solrcloud-autoscaling|suggestions-screen|uploading-structured-data-store-data-with-the-data-import-handler|velocity-response-writer|velocity-search-ui)\.html$#">- Header set X-Robots-Tag "noindex,nofollow,noarchive"-</If>diff --git a/dev-tools/scripts/refguide/mappings.csv b/dev-tools/scripts/refguide/mappings.csvindex 910207af4fb..94f6754b1ef 100644--- a/dev-tools/scripts/refguide/mappings.csv+++ b/dev-tools/scripts/refguide/mappings.csv@@ -19,6 +19,7 @@ indexconfig-in-solrconfig.html;index-segments-merging.htmlindexing-and-basic-data-operations.html;indexing-with-update-handlers.htmlinitparams-in-solrconfig.html;initparams.htmlintroduction-to-solr-indexing.html;solr-indexing.html+lib-directives-in-solrconfig.html;libs.htmllocal-parameters-in-queries.html;local-params.htmlmajor-changes-from-solr-5-to-solr-6.html;major-changes-in-solr-6.htmlmaking-and-restoring-backups.html;backup-restore.html@@ -105,6 +106,21 @@ the-well-configured-solr-instance.adoc;configuration-files.htmlsolrcloud.adoc;cluster-types.html#solrcloud-modehow-to-contribute.adoc;https://solr.apache.org/community.html#how-to-contributedeployment-and-operations.adoc;installing-solr.html+vectorization.adoc;math-expressions.html+time-routed-aliases.adoc;https://solr.apache.org/guide/solr/latest/deployment-guide/aliases.html#time-routed-aliases+resource-and-plugin-loading.adoc;resource-loading.html+getting-assistance.adoc;https://solr.apache.org/community.html+statistical-programming.adoc;math-expressions.html+lib-directives-in-solrconfig.adoc;libs.html+uima-integration.adoc;https://solr.apache.org/guide/7_4/uima-integration.html+upgrading-solr.adoc;upgrading-a-solr-cluster.html+stream-decorators.adoc;stream-decorator-reference.html+stream-sources.adoc;stream-source-reference.html+stream-evaluators.adoc;stream-evaluator-reference.html+a-step-closer.adoc;configuration-files.html+read-and-write-side-fault-tolerance.adoc;solrcloud-recoveries-and-write-tolerance.html+managing-solr.adoc;solr-control-script-reference.html+running-solr.adoc;installing-solr.html# A bit uncertain of theseparallel-sql-interface.html;sql-query.htmldiff --git a/dev-tools/scripts/refguide/old-guide.txt b/dev-tools/scripts/refguide/old-guide.txtindex cf1478100d6..7edc6f5fe1a 100644--- a/dev-tools/scripts/refguide/old-guide.txt+++ b/dev-tools/scripts/refguide/old-guide.txt@@ -1,4 +1,5 @@a-quick-overview.adoc+a-step-closer.adocabout-filters.adocabout-this-guide.adocabout-tokenizers.adoc@@ -77,6 +78,7 @@ filter-descriptions.adocformat-of-solr-xml.adocfunction-queries.adocfurther-assistance.adoc+getting-assistance.adocgetting-started-with-solrcloud.adocgetting-started.adocgraph-traversal.adoc@@ -107,6 +109,7 @@ kerberos-authentication-plugin.adoclanguage-analysis.adoclearning-to-rank.adoclegacy-scaling-and-distribution.adoc+lib-directives-in-solrconfig.adoclibs.adocloading.adoclocal-parameters-in-queries.adoc@@ -119,6 +122,7 @@ major-changes-in-solr-7.adocmajor-changes-in-solr-8.adocmaking-and-restoring-backups.adocmanaged-resources.adoc+managing-solr.adocmath-expressions.adocmath-start.adocmatrix-math.adoc@@ -153,6 +157,7 @@ query-re-ranking.adocquery-screen.adocquery-settings-in-solrconfig.adocquery-syntax-and-parsing.adoc+read-and-write-side-fault-tolerance.adocrealtime-get.adocregression.adocreindexing.adoc@@ -162,6 +167,7 @@ replication-screen.adocrequest-parameters-api.adocrequestdispatcher-in-solrconfig.adocrequesthandlers-and-searchcomponents-in-solrconfig.adoc+resource-and-plugin-loading.adocresource-loading.adocresponse-writers.adocresult-grouping.adoc@@ -169,6 +175,7 @@ rule-based-authorization-plugin.adocrule-based-replica-placement.adocrunning-solr-on-hdfs.adocrunning-your-analyzer.adoc+running-solr.adocscalar-math.adocschema-api.adocschema-browser-screen.adoc@@ -218,16 +225,21 @@ solrcloud.adocspatial-search.adocspell-checking.adocstatistics.adoc+statistical-programming.adocstream-api.adocstream-decorator-reference.adoc+stream-decorators.adocstream-evaluator-reference.adoc+stream-evaluators.adocstream-screen.adocstream-source-reference.adoc+stream-sources.adocstreaming-expressions.adocsuggester.adocsuggestions-screen.adoctaking-solr-to-production.adocterm-vectors.adoc+time-routed-aliases.adocthe-dismax-query-parser.adocthe-extended-dismax-query-parser.adocthe-query-elevation-component.adoc@@ -243,11 +255,13 @@ tokenizers.adoctransform.adoctransforming-and-indexing-custom-json.adoctransforming-result-documents.adoc+uima-integration.adocunderstanding-analyzers-tokenizers-and-filters.adocupdate-request-processors.adocupdatehandlers-in-solrconfig.adocupdating-parts-of-documents.adocupgrading-a-solr-cluster.adoc+upgrading-solr.adocuploading-data-with-index-handlers.adocuploading-data-with-solr-cell-using-apache-tika.adocuploading-structured-data-store-data-with-the-data-import-handler.adoc@@ -261,6 +275,7 @@ using-zookeeper-to-manage-configuration-files.adocv2-api.adocvariables.adocvector-math.adoc+vectorization.adocvelocity-response-writer.adocvelocity-search-ui.adocvisualization.adocdiff --git a/dev-tools/scripts/refguide/refguide-add-canonical-url.py b/dev-tools/scripts/refguide/refguide-add-canonical-url.pynew file mode 100755index 00000000000..d0ed8c7e941--- /dev/null+++ b/dev-tools/scripts/refguide/refguide-add-canonical-url.py@@ -0,0 +1,189 @@+#!/usr/bin/env python3+# -*- coding: utf-8 -*-+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+This script processes all static html files for Solr's refernce guide+and downloads external JS and CSS files to local folders js/ and css/ for+each version. It also updates the HTML files to reference the local files.+Context is that ASF policy for web sites changed to not allow external+references to JS and CSS files, and these sites were generated long ago.+"""++import os+import requests+from bs4 import BeautifulSoup+from urllib.parse import urlparse+import re+import argparse+++def lines_from_file(filename):+ with open(filename, 'r') as fp:+ lines = []+ for line in fp.readlines():+ if line.startswith("#") or len(line.strip()) == 0:+ continue+ lines.append(line.replace(".adoc", ".html").strip())+ return lines++def generate_canonical_mapping(conf):+ new = {}+ name_map = {}++ print("Reading config")+ old = lines_from_file(conf.old)+ for line in lines_from_file(conf.new):+ (path, file) = line.split("/")+ new[file] = line+ for line in lines_from_file(conf.mapping):+ (frm, to) = line.split(";")+ name_map[frm] = to++ # Files in src/old-pages as of 2022-02-04+ old_pages = ["configuration-apis.html", "configuration-guide.html", "controlling-results.html", "deployment-guide.html", "enhancing-queries.html", "field-types.html", "fields-and-schema-design.html", "getting-started.html", "indexing-data-operations.html", "installation-deployment.html", "monitoring-solr.html", "query-guide.html", "scaling-solr.html", "schema-indexing-guide.html", "solr-concepts.html", "solr-schema.html", "solrcloud-clusters.html", "user-managed-clusters.html"]++ result = {}+ old_guide = []+ failed = {}+ regex_new = {}+ print("Converting...")+ for frm in old:+ if frm in new:+ (subpath, name) = new[frm].split("/")+ if subpath not in regex_new:+ regex_new[subpath] = []+ regex_new[subpath].append(name.split(".html")[0])+ elif frm in name_map:+ new_name = name_map[frm]+ new_name_without_anchor = new_name+ anchor = ""+ anchor_index = new_name.find("#")+ if anchor_index > 0:+ new_name_without_anchor = new_name[:anchor_index]+ anchor = new_name[anchor_index:]+ if new_name_without_anchor.startswith("https://"):+ result[frm] = new_name+ elif new_name_without_anchor in new:+ result[frm] = new[new_name_without_anchor] + anchor+ elif new_name_without_anchor.startswith("/guide/"):+ result[frm] = new_name[7:]+ elif new_name_without_anchor == "_8_11":+ old_guide.append(frm.split(".html")[0])+ else:+ failed[frm] = "Mapped value %s not in new guide" % new_name_without_anchor+ elif frm in old_pages:+ failed[frm] = "Not yet mapped (in src/old-pages)"+ else:+ failed[frm] = "404"++ mappings = {+ "index.html": "https://solr.apache.org/guide/solr/latest/index.html",++ }+ # Add direct mappings from old to new files+ for key in regex_new:+ for file in regex_new[key]:+ mappings[file + ".html"] = f"https://solr.apache.org/guide/solr/latest/{key}/{file}.html"++ # Add mappings for renamed files+ for key in result:+ if result[key].startswith("https://"):+ mappings[key] = result[key]+ else:+ mappings[key] = f"https://solr.apache.org/guide/solr/latest/{result[key]}"++ # Add mappings for files removed in 9.0, they will be canonical to 8.11+ for file in old_guide:+ mappings[file + ".html"] = f"https://solr.apache.org/guide/8_11/{file}.html"++ for (key, value) in mappings.items():+ print(key, value)+ return mappings++def extract_filename_from_path(html_file_path):+ """Extract filename from path."""+ match = re.search(r'/([^\/]+)$', html_file_path)+ return match.group(1) if match else None++def process_html_file(html_file_path, url, mappings):+ """Process an HTML file to localize external JS and CSS references."""+ with open(html_file_path, "r", encoding="utf-8") as f:+ lines = f.readlines()++ file_name = extract_filename_from_path(html_file_path)+ if file_name and file_name in mappings:+ canonical_url = mappings[file_name]+ else:+ canonical_url = url+ if canonical_url == url:+ print(f"Skipped {html_file_path}, filename {file_name}, it is the canonical url: {url}")+ return+ canonical_link_html = f"<link rel=\"canonical\" href=\"{canonical_url}\">\n"++ new_lines = []+ found_title = False+ for line in lines:+ soup = BeautifulSoup(line, "html.parser")+ title = soup.find("title")+ canon_link = soup.find("link", attrs={'rel': 'canonical'})++ if title and not found_title:+ new_lines.append(line)+ new_lines.append(canonical_link_html)+ found_title = True+ elif not (found_title and canon_link):+ # Skip any other canonical url we find+ new_lines.append(line)++ if found_title:+ with open(html_file_path, "w", encoding="utf-8") as f:+ f.writelines(new_lines)+ print(f"Updated {html_file_path} to canonical url: {canonical_url}")++def main():+ parser = argparse.ArgumentParser(description='Process HTML files to add Canonical URLs to old ref guide pages')+ parser.add_argument('--old', required=True, help='Old pagenames file, one .adoc filename per line')+ parser.add_argument('--new', required=True, help='New pagenames file, one .adoc filename per line')+ parser.add_argument('--mapping', required=True, help='Semicolon separated from-to file names (adoc)')+ parser.add_argument('--folder', help='Folder of svn checkout (https://svn.apache.org/repos/infra/sites/solr/guide/)')+ args = parser.parse_args()+ mappings = generate_canonical_mapping(args)++ base_dir = args.folder++ # Iterate over the folder structure+ folders = [name for name in os.listdir(base_dir) if re.match(r'\d+_\d+', name)]+ if not folders:+ print(f"No versioned directories 'N_M' found in {base_dir}, exiting.")+ return+ for root_dir in folders:+ print(f"\nProcessing directory {root_dir}")+ print(f"=================================")+ full_path = os.path.join(base_dir, root_dir)+ if not os.path.exists(full_path):+ print(f"Directory {full_path} not found, skipping.")+ continue++ # Process each HTML file in the directory+ for filename in os.listdir(full_path):+ if filename.endswith(".html"):+ html_file_path = os.path.join(full_path, filename)+ url = f"https://solr.apache.org/guide/{root_dir}/{filename}"+ process_html_file(html_file_path, url, mappings)++if __name__ == "__main__":+ main()\ No newline at end of file