SearXNG is a free internet metasearch engine which aggregates results from various search services and databases. Users are neither tracked nor profiled.
Python bingbravedegoogleduckduckgogooglemetasearchprivacypythonqwantsearchsearch-enginesearxsearxngstartpageyahoo
[mod] add Golang ecosystem to the SearXNG toolchain
Markus Heiser20a193f04c8d
diff --git a/.gitignore b/.gitignoreindex 1305d0ac8..ea6147b1c 100644--- a/.gitignore+++ b/.gitignore@@ -9,6 +9,7 @@ geckodriver.log.coveragecoverage/+.govm/.nvm/cache/build/diff --git a/Makefile b/Makefileindex a4b19c25d..4c633e1fc 100644--- a/Makefile+++ b/Makefile@@ -32,7 +32,7 @@ install uninstall:$(Q)./manage pyenv.$@PHONY += clean-clean: py.clean docs.clean node.clean nvm.clean test.clean+clean: py.clean docs.clean node.clean nvm.clean go.clean test.clean$(Q)./manage build_msg CLEAN "common files"$(Q)find . -name '*.orig' -exec rm -f {} +$(Q)find . -name '*.rej' -exec rm -f {} +@@ -57,7 +57,7 @@ test.shell:$(MTOOLS) \utils/lib.sh \utils/lib_sxng*.sh \- utils/lib_go.sh \+ utils/lib_govm.sh \utils/lib_nvm.sh \utils/lib_redis.sh \utils/lib_valkey.sh \@@ -82,6 +82,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.coverMANAGE += themes.all themes.simple themes.fix themes.lint themes.testMANAGE += static.build.commit static.build.drop static.build.restoreMANAGE += nvm.install nvm.clean nvm.status nvm.nodejs+MANAGE += go.env.dev go.cleanPHONY += $(MANAGE)diff --git a/manage b/manageindex cf3abbca6..7d2305e65 100755--- a/manage+++ b/manage@@ -32,8 +32,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"# shellcheck source=utils/lib_sxng_test.shsource "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"-# shellcheck source=utils/lib_go.sh-source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"+# shellcheck source=utils/lib_govm.sh+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_govm.sh"# shellcheck source=utils/lib_valkey.shsource "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"@@ -41,7 +41,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"# shellcheck source=utils/lib_sxng_vite.shsource "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"-PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"+# add ./local dev tools from python (virtualenv), golang and nodejs+PATH="${PY_ENV}/bin:${REPO_ROOT}/node_modules/.bin:${GOROOT}/bin:${GOPATH}/bin:${PATH}"# config@@ -100,11 +101,27 @@ EOFstatic.helpvite.helpcat <<EOF+dev.:+ env: enter developer environment (or exec a command in)environment ...SEARXNG_VALKEY_URL : ${SEARXNG_VALKEY_URL}EOF}+dev.env() {+ go.env.dev+ node.env.dev++ export GOENV++ if [ -z "$1" ]; then+ export PS1="(dev.env)$ "+ bash --norc --noprofile+ else+ "$@"+ fi+}+if [ "$VERBOSE" = "1" ]; thenSPHINX_VERBOSE="-v"diff --git a/utils/lib_go.sh b/utils/lib_go.shdeleted file mode 100755index 06522f005..000000000--- a/utils/lib_go.sh+++ /dev/null@@ -1,211 +0,0 @@-#!/usr/bin/env bash-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*--# SPDX-License-Identifier: AGPL-3.0-or-later-#-# Tools to install and maintain golang [1] binaries & packages.-#-# [1] https://golang.org/doc/devel/release#policy-#-# A simple *helloworld* test with user 'my_user' :-#-# sudo -H adduser my_user-# ./manage go.golang go1.17.3 my_user-# ./manage go.install github.com/go-training/helloworld@latest my_user-# ./manage go.bash my_user-# $ helloword-# Hello World!!-#-# Don't forget to remove 'my_user': sudo -H deluser --remove-home my_user--# shellcheck source=utils/lib.sh-. /dev/null--# configure golang environment-# ------------------------------[[ -z "${GO_VERSION}" ]] && GO_VERSION="go1.17.3"--GO_DL_URL="https://golang.org/dl"--# implement go functions-# -------------------------go.help(){- cat <<EOF-go.:- ls : list golang binary archives (stable)- golang : (re-) install golang binary in user's \$HOME/local folder- install : install go package in user's \$HOME/go-apps folder- bash : start bash interpreter with golang environment sourced-EOF-}--go.ls(){- python <<EOF-import sys, json, requests-resp = requests.get("${GO_DL_URL}/?mode=json&include=all")-for ver in json.loads(resp.text):- if not ver['stable']:- continue- for f in ver['files']:- if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:- continue- print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)-EOF-}--go.ver_info(){-- # print information about a golang distribution. To print filename- # sha256 and size of the archive that fits to your OS and host:- #- # go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size- #- # usage: go.ver_info <go-vers> <kind> <os> <arch> [filename|sha256|size]- #- # kind: [archive|source|installer]- # os: [darwin|freebsd|linux|windows]- # arch: [amd64|arm64|386|armv6l|ppc64le|s390x]-- python - "$@" <<EOF-import sys, json, requests-resp = requests.get("${GO_DL_URL}/?mode=json&include=all")-for ver in json.loads(resp.text):- if ver['version'] != sys.argv[1]:- continue- for f in ver['files']:- if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):- continue- for x in sys.argv[5:]:- print(f[x])- sys.exit(0)-sys.exit(42)-EOF-}--go.os() {- local OS- case "$(command uname -a)xx" in- Linux\ *) OS=linux ;;- Darwin\ *) OS=darwin ;;- FreeBSD\ *) OS=freebsd ;;- CYGWIN* | MSYS* | MINGW*) OS=windows ;;- *) die 42 "OS is unknown: $(command uname -a)" ;;- esac- echo "${OS}"-}--go.arch() {- local ARCH- case "$(command uname -m)" in- "x86_64") ARCH=amd64 ;;- "aarch64") ARCH=arm64 ;;- "armv6" | "armv7l") ARCH=armv6l ;;- "armv8") ARCH=arm64 ;;- .*386.*) ARCH=386 ;;- ppc64*) ARCH=ppc64le ;;- *) die 42 "ARCH is unknown: $(command uname -m)" ;;- esac- echo "${ARCH}"-}--go.golang() {-- # install golang binary in user's $HOME/local folder:- #- # go.golang ${GO_VERSION} ${SERVICE_USER}- #- # usage: go.golang <go-vers> [<username>]-- local version fname sha size user userpr- local buf=()-- version="${1:-${GO_VERSION}}"- user="${2:-${USERNAME}}"- userpr=" ${_Yellow}|${user}|${_creset} "-- rst_title "Install Go in ${user}'s HOME" section-- mapfile -t buf < <(- go.ver_info "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size- )-- if [ ${#buf[@]} -eq 0 ]; then- die 42 "can't find info of golang version: ${version}"- fi- fname="${buf[0]}"- sha="${buf[1]}"- size="$(numfmt --to=iec "${buf[2]}")"-- info_msg "Download go binary ${fname} (${size}B)"- cache_download "${GO_DL_URL}/${fname}" "${fname}"-- pushd "${CACHE}" &> /dev/null- echo "${sha} ${fname}" > "${fname}.sha256"- if ! sha256sum -c "${fname}.sha256" >/dev/null; then- die 42 "downloaded file ${fname} checksum does not match"- else- info_msg "${fname} checksum OK"- fi- popd &> /dev/null-- info_msg "install golang"- tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"-mkdir -p \$HOME/local-rm -rf \$HOME/local/go-tar -C \$HOME/local -xzf ${CACHE}/${fname}-echo "export GOPATH=\$HOME/go-apps" > \$HOME/.go_env-echo "export PATH=\$HOME/local/go/bin:\\\$GOPATH/bin:\\\$PATH" >> \$HOME/.go_env-EOF- info_msg "test golang installation"- sudo -i -u "${user}" <<EOF-source \$HOME/.go_env-command -v go-go version-EOF-}--go.install() {-- # install go package in user's $HOME/go-apps folder:- #- # go.install github.com/go-training/helloworld@lates ${SERVICE_USER}- #- # usage: go.install <package> [<username>]-- local package user userpr-- package="${1}"- user="${2:-${USERNAME}}"- userpr=" ${_Yellow}|${user}|${_creset} "-- if [ -z "${package}" ]; then- die 42 "${FUNCNAME[0]}() - missing argument: <package>"- fi- tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"-source \$HOME/.go_env-go install -v ${package}-EOF-}--go.bash() {-- # start bash interpreter with golang environment sourced- #- # go.bash ${SERVICE_USER}- #- # usage: go.bash [<username>]-- local user- user="${1:-${USERNAME}}"- sudo -i -u "${user}" bash --init-file "~${user}/.go_env"-}--go.version(){- local user- user="${1:-${USERNAME}}"- sudo -i -u "${user}" <<EOF-source \$HOME/.go_env-go version | cut -d' ' -f 3-EOF-}diff --git a/utils/lib_govm.sh b/utils/lib_govm.shnew file mode 100755index 000000000..c74f4a65a--- /dev/null+++ b/utils/lib_govm.sh@@ -0,0 +1,226 @@+#!/usr/bin/env bash+# SPDX-License-Identifier: AGPL-3.0-or-later+#+# Go versions manager to install and maintain golang [1] binaries & packages.+#+# [1] https://golang.org/doc/devel/release#policy++# shellcheck source=utils/lib.sh+. /dev/null++# configure golang environment for go.vm+# --------------------------------------++_GO_DL_URL="https://go.dev/dl"++GOVERSION="${GOVERSION:-go$(awk '/^go /{print $2}' "${REPO_ROOT}/go.mod")}"+GOTOOLCHAIN=local++GOROOT="${REPO_ROOT}/.govm/${GOVERSION}"+GOENV="${GOROOT}/.config/go.env"+GOVM_EXE="${GOROOT}/bin/go"++GOPATH="${REPO_ROOT}/local/${GOVERSION}" # no support for multiple path names!!+GOCACHE="${GOPATH}/.cache/go-build"+GOMODCACHE="${GOPATH}/pkg/mod"++# implement go functions+# -----------------------++go.help() {+ cat <<EOF+go: GOROOT=${GOROOT}+ install : compiles and installs packages+EOF+}++go.tool() {+ # shortcut for "go tool .." in the Go environment+ go.env.dev+ "${GOVM_EXE}" tool "$@"+}++go.env.dev() {+ if [ -z "$_GO_DEVTOOLS_INSTALLED" ]; then+ build_msg INSTALL "[pkg.go.dev] ./go.mod: developer and CI tools"+ go.tidy+ else+ go.vm.ensure+ _GO_DEVTOOLS_INSTALLED=1+ fi+}++go.tidy() {+ go.vm.ensure+ "${GOVM_EXE}" mod tidy+ chmod -R u+w "${GOMODCACHE}"+}++go.clean() {+ if ! go.vm.is_installed; then+ build_msg CLEAN "[Go] not installed"+ return 0+ fi+ build_msg CLEAN "[Go] drop folders ${GOROOT} and ${GOPATH}"+ rm -rf "${GOROOT}" "${GOPATH}"+}++go.install() {+ go.vm.ensure+ GOENV="${GOENV}" "${GOVM_EXE}" install "$@"+ # not sure why, but go installs some files without setting the write access+ # for the file owner+ chmod -R u+w "${GOMODCACHE}"+}++go.os() {+ local OS+ case "$(command uname -a)xx" in+ Linux\ *) OS=linux ;;+ Darwin\ *) OS=darwin ;;+ FreeBSD\ *) OS=freebsd ;;+ CYGWIN* | MSYS* | MINGW*) OS=windows ;;+ *) die 42 "OS is unknown: $(command uname -a)" ;;+ esac+ echo "${OS}"+}++go.arch() {+ local ARCH+ case "$(command uname -m)" in+ "x86_64") ARCH=amd64 ;;+ "aarch64") ARCH=arm64 ;;+ "armv6" | "armv7l") ARCH=armv6l ;;+ "armv8") ARCH=arm64 ;;+ .*386.*) ARCH=386 ;;+ ppc64*) ARCH=ppc64le ;;+ *) die 42 "ARCH is unknown: $(command uname -m)" ;;+ esac+ echo "${ARCH}"+}++# Go version management (go.vm)+# -----------------------------++go.vm.ensure() {+ if ! go.vm.is_installed; then+ # shellcheck disable=SC2119+ go.vm.install+ fi+}++go.vm.is_installed() {+ # is true if "go" command is installed+ [[ -f "${GOROOT}/bin/go" ]]+}++# shellcheck disable=SC2120+go.vm.install() {++ python -m pip install -U requests++ # Go versions manager; to install Go at arbitrary place:+ #+ # usage: go.vm.install <version> <dest>++ local version dest fname sha size tmp+ version="${1:-$GOVERSION}"+ dest="${2:-$GOROOT}"++ info_msg "Install Go in ${dest}"++ # HINT: the python requirements needed by go.vm.version are taken from the+ # developer environment. If it is not yet installed, install it now ..+ pyenv.install++ # fetch go version ..+ local buf=()+ mapfile -t buf < <(+ go.vm.version "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size+ )+ if [ ${#buf[@]} -eq 0 ]; then+ die 42 "can't find info of golang version: ${version}"+ fi+ fname="${buf[0]}"+ sha="${buf[1]}"+ size="$(numfmt --to=iec "${buf[2]}")"++ info_msg "Download go binary ${fname} (${size}B)"+ cache_download "${_GO_DL_URL}/${fname}" "${fname}"++ pushd "${CACHE}" &>/dev/null+ echo "${sha} ${fname}" >"${fname}.sha256"+ if ! sha256sum -c "${fname}.sha256" >/dev/null; then+ die 42 "downloaded file ${fname} checksum does not match"+ else+ info_msg "${fname} checksum OK"+ fi+ popd &>/dev/null++ info_msg "install golang"++ tmp="$(mktemp -d)"+ tar -C "${tmp}" -xzf "${CACHE}/${fname}"+ rm -rf "${dest}"+ mkdir -p "$(dirname "${dest}")"+ mv "${tmp}/go" "${dest}"++ mkdir -p "$(dirname "$GOENV")"+ export GOENV++ "${GOVM_EXE}" telemetry off+ "${GOVM_EXE}" env -w \+ GOBIN="$GOBIN" \+ GOTOOLCHAIN="$GOTOOLCHAIN" \+ GOCACHE="$GOCACHE" \+ GOPATH="$GOPATH" \+ GOMODCACHE="$GOMODCACHE"++ mkdir -p "${GOMODCACHE}"+}++go.vm.list() {++ # Go versions manager; list Go versions (stable)++ python <<EOF+import sys, json, requests+resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")+for ver in json.loads(resp.text):+ if not ver['stable']:+ continue+ for f in ver['files']:+ if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:+ continue+ print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)+EOF+}++go.vm.version() {++ # Print information about a Go distribution. To print filename sha256 and+ # size of the archive that fits to your OS and host:+ #+ # go.ver_info "${GOVERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size+ #+ # usage: go.vm.version <go-vers> <kind> <os> <arch> [filename|sha256|size]+ #+ # kind: [archive|source|installer]+ # os: [darwin|freebsd|linux|windows]+ # arch: [amd64|arm64|386|armv6l|ppc64le|s390x]++ python - "$@" <<EOF+import sys, json, requests+resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")+for ver in json.loads(resp.text):+ if ver['version'] != sys.argv[1]:+ continue+ for f in ver['files']:+ if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):+ continue+ for x in sys.argv[5:]:+ print(f[x])+ sys.exit(0)+sys.exit(42)+EOF+}