LLM inference in C/C++
C++ ggml
vendor : apply patches for subprocess.h (#26606)
Xuan-Son Nguyen61881b1f7f0b
diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.pyindex 9faa6a307c..98840ac724 100755--- a/scripts/sync_vendor.py+++ b/scripts/sync_vendor.py@@ -24,10 +24,31 @@ vendor = {"https://raw.githubusercontent.com/sheredom/subprocess.h/8671cee1fc09f11a70ce3782a0ee13177c3aa387/subprocess.h": "vendor/sheredom/subprocess.h",}+# TODO @ngxson : this is temporary, to be removed in the future+patches = [+ # https://github.com/sheredom/subprocess.h/pull/102+ "vendor/sheredom/patch-bsd.patch",+ # https://github.com/sheredom/subprocess.h/pull/101+ "vendor/sheredom/patch-windows-quote-backslash.patch",+ # https://github.com/sheredom/subprocess.h/pull/104+ # note: must be applied after patch-bsd.patch, they touch adjacent lines+ "vendor/sheredom/patch-glibc-older-than-2.29.patch",+]+for url, filename in vendor.items():print(f"downloading {url} to {filename}") # noqa: NP100urllib.request.urlretrieve(url, filename)+for patch in patches:+ print(f"applying {patch}") # noqa: NP100+ try:+ subprocess.check_call([+ "git", "apply", "--directory", os.path.dirname(patch), patch+ ])+ except Exception as e:+ print(f"Error: {e}") # noqa: NP100+ sys.exit(1)+print("Splitting httplib.h...") # noqa: NP100try:subprocess.check_call([diff --git a/vendor/sheredom/patch-bsd.patch b/vendor/sheredom/patch-bsd.patchnew file mode 100644index 0000000000..2532050e20--- /dev/null+++ b/vendor/sheredom/patch-bsd.patch@@ -0,0 +1,19 @@+Fix build on NetBSD, which provides posix_spawn_file_actions_addchdir()+but not the _np() variant.++Upstream PR: https://github.com/sheredom/subprocess.h/pull/102+Applied locally by scripts/sync_vendor.py until it is merged upstream.++diff --git a/subprocess.h b/subprocess.h+index 5e809023a4..74a4e006c7 100644+--- a/subprocess.h++++ b/subprocess.h+@@ -1205,7 +1205,7 @@ cleanup:++ // Set working directory+ if (process_cwd) {+-#if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000++#if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)+ posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);+ #else+ #if defined(__APPLE__) && defined(__clang__)diff --git a/vendor/sheredom/patch-glibc-older-than-2.29.patch b/vendor/sheredom/patch-glibc-older-than-2.29.patchnew file mode 100644index 0000000000..9144213096--- /dev/null+++ b/vendor/sheredom/patch-glibc-older-than-2.29.patch@@ -0,0 +1,47 @@+Fix building against glibc older than 2.29, which has no+posix_spawn_file_actions_addchdir_np (the symbol is genuinely absent from+libc.so, so no feature-test macro helps). Affects manylinux2014 (glibc 2.17)+and manylinux_2_28, and was reported on RHEL 8.1. A requested process_cwd now+fails with ENOSYS there instead of failing the build.++Upstream PR: https://github.com/sheredom/subprocess.h/pull/104+Applied locally by scripts/sync_vendor.py until it is merged upstream.++(the README.md and test/ changes from the PR are omitted, we only vendor+subprocess.h; rebased on top of patch-bsd.patch, so apply it after that one)++diff --git a/subprocess.h b/subprocess.h+index 1ef424a..c363393 100644+--- a/subprocess.h++++ b/subprocess.h+@@ -274,6 +274,21 @@ subprocess_weak int subprocess_alive(struct subprocess_s *const process);+ #include <unistd.h>+ #endif+++/* Whether subprocess_create_ex can honour process_cwd. glibc only gained++ posix_spawn_file_actions_addchdir_np in 2.29. Define this yourself to++ override the detection, for instance on musl older than 1.1.24. */++#if !defined(SUBPROCESS_HAVE_CWD)++#if defined(__GLIBC__)++#if __GLIBC_PREREQ(2, 29)++#define SUBPROCESS_HAVE_CWD 1++#else++#define SUBPROCESS_HAVE_CWD 0++#endif++#else++#define SUBPROCESS_HAVE_CWD 1++#endif++#endif+++ #if defined(_WIN32)++ #include <wchar.h>+@@ -1219,6 +1234,8 @@ cleanup:+ if (process_cwd) {+ #if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)+ posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);++#elif !SUBPROCESS_HAVE_CWD++ posix_error = ENOSYS;+ #else+ #if defined(__APPLE__) && defined(__clang__)+ #pragma clang diagnostic pushdiff --git a/vendor/sheredom/patch-windows-quote-backslash.patch b/vendor/sheredom/patch-windows-quote-backslash.patchnew file mode 100644index 0000000000..0204746d96--- /dev/null+++ b/vendor/sheredom/patch-windows-quote-backslash.patch@@ -0,0 +1,107 @@+Fix Windows command line quoting of backslash runs: a trailing backslash, or+backslashes preceding a double quote, were not doubled, so CommandLineToArgvW+in the child parsed them as escapes and mangled the argument list.++Upstream PR: https://github.com/sheredom/subprocess.h/pull/101+Applied locally by scripts/sync_vendor.py until it is merged upstream.++(the test/ changes from the PR are omitted, we only vendor subprocess.h)++diff --git a/subprocess.h b/subprocess.h+index 5e80902..b06ad4d 100644+--- a/subprocess.h++++ b/subprocess.h+@@ -653,6 +653,7 @@ int subprocess_create_ex(const char *const commandLine[], int options,+ int wide_len;+ int i, j;+ int need_quoting;++ subprocess_size_t bs_run;+ unsigned long flags = 0;+ unsigned long last_error = 0;+ int result = subprocess_error_unknown;+@@ -906,25 +907,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,+ len++;++ // Quote the argument if it has a space in it+- if (strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||+- commandLine[i][0] == SUBPROCESS_NULL)++ need_quoting = strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||++ commandLine[i][0] == SUBPROCESS_NULL;++ if (need_quoting)+ len += 2;+++ bs_run = 0;+ for (j = 0; '\0' != commandLine[i][j]; j++) {+- switch (commandLine[i][j]) {+- default:+- break;+- case '\\':+- if (commandLine[i][j + 1] == '"') {+- len++;+- }++ len++;++- break;+- case '"':+- len++;+- break;++ if ('\\' == commandLine[i][j]) {++ bs_run++;++ } else {++ if ('"' == commandLine[i][j]) {++ // Duplicate the preceding run and escape the quote.++ len += bs_run + 1;++ }++ bs_run = 0;+ }+- len++;++ }++++ if (need_quoting) {++ // Duplicate trailing slashes before the generated closing quote.++ len += bs_run;+ }+ }++@@ -949,22 +954,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,+ commandLineCombined[len++] = '"';+ }++- for (j = 0; '\0' != commandLine[i][j]; j++) {+- switch (commandLine[i][j]) {+- default:+- break;+- case '\\':+- if (commandLine[i][j + 1] == '"') {+- commandLineCombined[len++] = '\\';+- }++ for (j = 0; '\0' != commandLine[i][j];) {++ bs_run = 0;++ while ('\\' == commandLine[i][j]) {++ bs_run++;++ j++;++ }++++ if ('"' == commandLine[i][j]) {++ // 2n + 1 slashes preserve n slashes and escape the quote.++ bs_run = (bs_run * 2) + 1;++ } else if ('\0' == commandLine[i][j] && need_quoting) {++ // 2n slashes preserve n slashes before the closing quote.++ bs_run *= 2;++ }++- break;+- case '"':++ while (bs_run > 0) {+ commandLineCombined[len++] = '\\';+- break;++ bs_run--;+ }++- commandLineCombined[len++] = commandLine[i][j];++ if ('\0' != commandLine[i][j]) {++ commandLineCombined[len++] = commandLine[i][j++];++ }+ }+ if (need_quoting) {+ commandLineCombined[len++] = '"';diff --git a/vendor/sheredom/subprocess.h b/vendor/sheredom/subprocess.hindex 5e809023a4..c363393873 100644--- a/vendor/sheredom/subprocess.h+++ b/vendor/sheredom/subprocess.h@@ -274,6 +274,21 @@ subprocess_weak int subprocess_alive(struct subprocess_s *const process);#include <unistd.h>#endif+/* Whether subprocess_create_ex can honour process_cwd. glibc only gained+ posix_spawn_file_actions_addchdir_np in 2.29. Define this yourself to+ override the detection, for instance on musl older than 1.1.24. */+#if !defined(SUBPROCESS_HAVE_CWD)+#if defined(__GLIBC__)+#if __GLIBC_PREREQ(2, 29)+#define SUBPROCESS_HAVE_CWD 1+#else+#define SUBPROCESS_HAVE_CWD 0+#endif+#else+#define SUBPROCESS_HAVE_CWD 1+#endif+#endif+#if defined(_WIN32)#include <wchar.h>@@ -653,6 +668,7 @@ int subprocess_create_ex(const char *const commandLine[], int options,int wide_len;int i, j;int need_quoting;+ subprocess_size_t bs_run;unsigned long flags = 0;unsigned long last_error = 0;int result = subprocess_error_unknown;@@ -906,25 +922,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,len++;// Quote the argument if it has a space in it- if (strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||- commandLine[i][0] == SUBPROCESS_NULL)+ need_quoting = strpbrk(commandLine[i], "\t\v ") != SUBPROCESS_NULL ||+ commandLine[i][0] == SUBPROCESS_NULL;+ if (need_quoting)len += 2;+ bs_run = 0;for (j = 0; '\0' != commandLine[i][j]; j++) {- switch (commandLine[i][j]) {- default:- break;- case '\\':- if (commandLine[i][j + 1] == '"') {- len++;- }+ len++;- break;- case '"':- len++;- break;+ if ('\\' == commandLine[i][j]) {+ bs_run++;+ } else {+ if ('"' == commandLine[i][j]) {+ // Duplicate the preceding run and escape the quote.+ len += bs_run + 1;+ }+ bs_run = 0;}- len++;+ }++ if (need_quoting) {+ // Duplicate trailing slashes before the generated closing quote.+ len += bs_run;}}@@ -949,22 +969,29 @@ int subprocess_create_ex(const char *const commandLine[], int options,commandLineCombined[len++] = '"';}- for (j = 0; '\0' != commandLine[i][j]; j++) {- switch (commandLine[i][j]) {- default:- break;- case '\\':- if (commandLine[i][j + 1] == '"') {- commandLineCombined[len++] = '\\';- }+ for (j = 0; '\0' != commandLine[i][j];) {+ bs_run = 0;+ while ('\\' == commandLine[i][j]) {+ bs_run++;+ j++;+ }- break;- case '"':+ if ('"' == commandLine[i][j]) {+ // 2n + 1 slashes preserve n slashes and escape the quote.+ bs_run = (bs_run * 2) + 1;+ } else if ('\0' == commandLine[i][j] && need_quoting) {+ // 2n slashes preserve n slashes before the closing quote.+ bs_run *= 2;+ }++ while (bs_run > 0) {commandLineCombined[len++] = '\\';- break;+ bs_run--;}- commandLineCombined[len++] = commandLine[i][j];+ if ('\0' != commandLine[i][j]) {+ commandLineCombined[len++] = commandLine[i][j++];+ }}if (need_quoting) {commandLineCombined[len++] = '"';@@ -1205,8 +1232,10 @@ cleanup:// Set working directoryif (process_cwd) {-#if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000+#if defined(__NetBSD__) || (defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000)posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd);+#elif !SUBPROCESS_HAVE_CWD+ posix_error = ENOSYS;#else#if defined(__APPLE__) && defined(__clang__)#pragma clang diagnostic push