Apache Solr open-source search software
SOLR-15728: Remove dead, unused log rotation code from SolrCLI (#375)
Jan Høydahlab9bec6aed30
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 92b907ea5e7..118b51a5fd5 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -384,6 +384,8 @@ Other Changes* SOLR-15699: Removal of the (unused since 5.0) "updateHandler/indexWriter/closeWaitsForMerges" solrconfig.xml element. (Christine Poerschke)+* SOLR-15728: Remove dead, unused log rotation code from SolrCLI (janhoy)+Bug Fixes---------------------* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order executiondiff --git a/solr/bin/solr b/solr/bin/solrindex 52a84c10ba8..0c46a265015 100755--- a/solr/bin/solr+++ b/solr/bin/solr@@ -1996,14 +1996,6 @@ if $verbose ; thenelseq="-q"fi-# Rotate and clean on startup. Default to false since 7.4 as log4j2 handles startup rotation-if [ "${SOLR_LOG_PRESTART_ROTATION:=false}" == "true" ]; then- # Enable any of these if you require old remove/archive behavior- #run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -remove_old_solr_logs 7 || echo "Failed removing old solr logs"- #run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -archive_gc_logs $q || echo "Failed archiving old GC logs"- #run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -archive_console_logs || echo "Failed archiving old console logs"- run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -rotate_solr_logs 10 || echo "Failed rotating old solr logs"-fi# Establish default GC logging opts if no env var set (otherwise init to sensible default)if [ -z ${GC_LOG_OPTS+x} ]; thendiff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmdindex e4f623da4e7..652595ad6f3 100755--- a/solr/bin/solr.cmd+++ b/solr/bin/solr.cmd@@ -1136,18 +1136,6 @@ if !JAVA_MAJOR_VERSION! LSS 9 (set IS_64bit=true)-REM Clean up and rotate logs. Default to false since 7.4 as log4j2 handles startup rotation-IF [%SOLR_LOG_PRESTART_ROTATION%] == [] (- set SOLR_LOG_PRESTART_ROTATION=false-)-IF [%SOLR_LOG_PRESTART_ROTATION%] == [true] (- REM Enable any of these if you require old remove/archive behavior- REM call :run_utils "-remove_old_solr_logs 7" || echo "Failed removing old solr logs"- REM call :run_utils "-archive_gc_logs" || echo "Failed archiving old GC logs"- REM call :run_utils "-archive_console_logs" || echo "Failed archiving old console logs"- call :run_utils "-rotate_solr_logs 9" || echo "Failed rotating old solr logs"-)-IF NOT "%ZK_HOST%"=="" set SOLR_MODE=solrcloudIF "%SOLR_MODE%"=="solrcloud" (diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmdindex 28d1bfe141b..b344879232e 100755--- a/solr/bin/solr.in.cmd+++ b/solr/bin/solr.in.cmd@@ -101,11 +101,6 @@ REM set SOLR_LOG_LEVEL=INFOREM Location where Solr should write logs to. Absolute or relative to solr start dirREM set SOLR_LOGS_DIR=logs-REM Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre-REM start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log-REM framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.-REM set SOLR_LOG_PRESTART_ROTATION=false-REM Enables jetty request log for all requestsREM set SOLR_REQUESTLOG_ENABLED=falsediff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.shindex 2d22ce5a65f..e8ef93cb0a8 100644--- a/solr/bin/solr.in.sh+++ b/solr/bin/solr.in.sh@@ -125,11 +125,6 @@# Location where Solr should write logs to. Absolute or relative to solr start dir#SOLR_LOGS_DIR=logs-# Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre-# start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log-# framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.-#SOLR_LOG_PRESTART_ROTATION=false-# Enables jetty request log for all requests#SOLR_REQUESTLOG_ENABLED=falsediff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.javaindex ef91d7ab882..ddaa7b525a1 100755--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java@@ -16,48 +16,6 @@*/package org.apache.solr.util;-import javax.net.ssl.SSLPeerUnverifiedException;-import java.io.Console;-import java.io.File;-import java.io.FileNotFoundException;-import java.io.IOException;-import java.io.InputStream;-import java.io.PrintStream;-import java.lang.invoke.MethodHandles;-import java.net.ConnectException;-import java.net.Socket;-import java.net.SocketException;-import java.net.URL;-import java.nio.charset.StandardCharsets;-import java.nio.file.Files;-import java.nio.file.Path;-import java.nio.file.Paths;-import java.nio.file.StandardCopyOption;-import java.nio.file.attribute.FileOwnerAttributeView;-import java.time.Instant;-import java.time.Period;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Base64;-import java.util.Collection;-import java.util.Collections;-import java.util.Enumeration;-import java.util.HashMap;-import java.util.LinkedHashMap;-import java.util.List;-import java.util.Locale;-import java.util.Map;-import java.util.Optional;-import java.util.Scanner;-import java.util.Set;-import java.util.TreeSet;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.TimeoutException;-import java.util.stream.Collectors;-import java.util.stream.Stream;-import java.util.zip.ZipEntry;-import java.util.zip.ZipInputStream;-import com.google.common.annotations.VisibleForTesting;import org.apache.commons.cli.CommandLine;import org.apache.commons.cli.GnuParser;@@ -103,13 +61,13 @@ import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;-import org.apache.solr.common.cloud.UrlScheme;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.cloud.UrlScheme;import org.apache.solr.common.cloud.ZkCoreNodeProps;-import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.cloud.ZkMaintenanceUtils;+import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;@@ -125,6 +83,43 @@ import org.noggit.JSONWriter;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import javax.net.ssl.SSLPeerUnverifiedException;+import java.io.Console;+import java.io.File;+import java.io.FileNotFoundException;+import java.io.IOException;+import java.io.InputStream;+import java.io.PrintStream;+import java.lang.invoke.MethodHandles;+import java.net.ConnectException;+import java.net.Socket;+import java.net.SocketException;+import java.net.URL;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.Paths;+import java.nio.file.attribute.FileOwnerAttributeView;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Base64;+import java.util.Collection;+import java.util.Collections;+import java.util.Enumeration;+import java.util.HashMap;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Optional;+import java.util.Scanner;+import java.util.Set;+import java.util.TreeSet;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.zip.ZipEntry;+import java.util.zip.ZipInputStream;+import static org.apache.solr.common.SolrException.ErrorCode.FORBIDDEN;import static org.apache.solr.common.SolrException.ErrorCode.UNAUTHORIZED;import static org.apache.solr.common.params.CommonParams.DISTRIB;@@ -4201,24 +4196,6 @@ public class SolrCLI implements CLIO {.build(),Option.builder("q").desc("Be quiet, don't print to stdout, only return exit codes.")- .build(),- Option.builder("remove_old_solr_logs")- .argName("daysToKeep")- .hasArg()- .type(Integer.class)- .desc("Path to logs directory.")- .build(),- Option.builder("rotate_solr_logs")- .argName("generations")- .hasArg()- .type(Integer.class)- .desc("Rotate solr.log to solr.log.1 etc.")- .build(),- Option.builder("archive_gc_logs")- .desc("Archive old garbage collection logs into archive/.")- .build(),- Option.builder("archive_console_logs")- .desc("Archive old console logs into archive/.").build()};}@@ -4238,145 +4215,9 @@ public class SolrCLI implements CLIO {if (cli.hasOption("q")) {beQuiet = cli.hasOption("q");}- if (cli.hasOption("remove_old_solr_logs")) {- if (removeOldSolrLogs(Integer.parseInt(cli.getOptionValue("remove_old_solr_logs"))) > 0) return 1;- }- if (cli.hasOption("rotate_solr_logs")) {- if (rotateSolrLogs(Integer.parseInt(cli.getOptionValue("rotate_solr_logs"))) > 0) return 1;- }- if (cli.hasOption("archive_gc_logs")) {- if (archiveGcLogs() > 0) return 1;- }- if (cli.hasOption("archive_console_logs")) {- if (archiveConsoleLogs() > 0) return 1;- }return 0;}- /**- * Moves gc logs into archived/- * @return 0 on success- * @throws Exception on failure- */- public int archiveGcLogs() throws Exception {- prepareLogsPath();- Path archivePath = logsPath.resolve("archived");- if (!archivePath.toFile().exists()) {- Files.createDirectories(archivePath);- }- List<Path> archived = Files.find(archivePath, 1, (f, a)- -> a.isRegularFile() && String.valueOf(f.getFileName()).matches("^solr_gc[_.].+"))- .collect(Collectors.toList());- for (Path p : archived) {- Files.delete(p);- }- List<Path> files = Files.find(logsPath, 1, (f, a)- -> a.isRegularFile() && String.valueOf(f.getFileName()).matches("^solr_gc[_.].+"))- .collect(Collectors.toList());- if (files.size() > 0) {- out("Archiving " + files.size() + " old GC log files to " + archivePath);- for (Path p : files) {- Files.move(p, archivePath.resolve(p.getFileName()), StandardCopyOption.REPLACE_EXISTING);- }- }- return 0;- }-- /**- * Moves console log(s) into archiced/- * @return 0 on success- * @throws Exception on failure- */- public int archiveConsoleLogs() throws Exception {- prepareLogsPath();- Path archivePath = logsPath.resolve("archived");- if (!archivePath.toFile().exists()) {- Files.createDirectories(archivePath);- }- List<Path> archived = Files.find(archivePath, 1, (f, a)- -> a.isRegularFile() && String.valueOf(f.getFileName()).endsWith("-console.log"))- .collect(Collectors.toList());- for (Path p : archived) {- Files.delete(p);- }- List<Path> files = Files.find(logsPath, 1, (f, a)- -> a.isRegularFile() && String.valueOf(f.getFileName()).endsWith("-console.log"))- .collect(Collectors.toList());- if (files.size() > 0) {- out("Archiving " + files.size() + " console log files to " + archivePath);- for (Path p : files) {- Files.move(p, archivePath.resolve(p.getFileName()), StandardCopyOption.REPLACE_EXISTING);- }- }- return 0;- }-- /**- * Rotates solr.log before starting Solr. Mimics log4j2 behavior, i.e. with generations=9:- * <pre>- * solr.log.9 (and higher) are deleted- * solr.log.8 -> solr.log.9- * solr.log.7 -> solr.log.8- * ...- * solr.log -> solr.log.1- * </pre>- * @param generations number of generations to keep. Should agree with setting in log4j2.xml- * @return 0 if success- * @throws Exception if problems- */- public int rotateSolrLogs(int generations) throws Exception {- prepareLogsPath();- if (logsPath.toFile().exists() && logsPath.resolve("solr.log").toFile().exists()) {- out("Rotating solr logs, keeping a max of "+generations+" generations.");- try (Stream<Path> files = Files.find(logsPath, 1,- (f, a) -> a.isRegularFile() && String.valueOf(f.getFileName()).startsWith("solr.log."))- .sorted((b,a) -> Integer.valueOf(a.getFileName().toString().substring(9))- .compareTo(Integer.valueOf(b.getFileName().toString().substring(9))))) {- files.forEach(p -> {- try {- int number = Integer.parseInt(p.getFileName().toString().substring(9));- if (number >= generations) {- Files.delete(p);- } else {- Path renamed = p.getParent().resolve("solr.log." + (number + 1));- Files.move(p, renamed);- }- } catch (IOException e) {- out("Problem during rotation of log files: " + e.getMessage());- }- });- } catch (NumberFormatException nfe) {- throw new Exception("Do not know how to rotate solr.log.<ext> with non-numeric extension. Rotate aborted.", nfe);- }- Files.move(logsPath.resolve("solr.log"), logsPath.resolve("solr.log.1"));- }-- return 0;- }-- /**- * Deletes time-stamped old solr logs, if older than n days- * @param daysToKeep number of days logs to keep before deleting- * @return 0 on success- * @throws Exception on failure- */- public int removeOldSolrLogs(int daysToKeep) throws Exception {- prepareLogsPath();- if (logsPath.toFile().exists()) {- try (Stream<Path> stream = Files.find(logsPath, 2, (f, a) -> a.isRegularFile()- && Instant.now().minus(Period.ofDays(daysToKeep)).isAfter(a.lastModifiedTime().toInstant())- && String.valueOf(f.getFileName()).startsWith("solr_log_"))) {- List<Path> files = stream.collect(Collectors.toList());- if (files.size() > 0) {- out("Deleting "+files.size() + " solr_log_* files older than " + daysToKeep + " days.");- for (Path p : files) {- Files.delete(p);- }- }- }- }- return 0;- }// Private methods to followdiff --git a/solr/solr-ref-guide/src/configuring-logging.adoc b/solr/solr-ref-guide/src/configuring-logging.adocindex 1acaa39cafa..21c658ec0c4 100644--- a/solr/solr-ref-guide/src/configuring-logging.adoc+++ b/solr/solr-ref-guide/src/configuring-logging.adoc@@ -126,7 +126,6 @@ Also, in `log4j2.xml` if the default log rotation size threshold of 32MB is tooJava Garbage Collection logs are rotated by the JVM when size hits 20M, for a max of 9 generations.On every startup or restart of Solr, log4j2 performs log rotation.-If you choose to use another log framework that does not support rotation on startup, you may enable `SOLR_LOG_PRESTART_ROTATION` in `bin/solr.in.sh` or `bin/solr.in.cmd` to let the start script rotate the logs on startup.== Logging Slow Queries