Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): validate index-creation, mapping, alias and template requests like ES (#1870)

Amartya Vikram Singh4df4eba1e6e4
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 42d84c48d..118988d04 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -1075,10 +1075,17 @@ pub async fn create_index(
return ApiError::new(e).into_response();
}
+ if let Some(refusal) = create_index_body_refusal(&index, &body) {
+ return refusal;
+ }
+
// Validate mapping field types before creating — ES returns
// mapper_parsing_exception for unsupported types.
let mappings_val = body.get("mappings").cloned().unwrap_or(Value::Null);
if let Some(props) = mappings_val.get("properties").and_then(Value::as_object) {
+ if let Some(refusal) = empty_field_name_refusal(props) {
+ return refusal;
+ }
if let Err(reason) = validate_properties(props) {
let e = xerj_common::XerjError::invalid_mapping(reason);
return ApiError::new(e).into_response();
@@ -1251,6 +1258,185 @@ pub async fn create_index(
}
}
+/// A 400 in Elasticsearch's error shape, for refusals whose `type` (or
+/// `caused_by`) a caller matches on and that no `XerjError` variant renders.
+fn es_bad_request(error_type: &str, reason: &str, caused_by: Option<(&str, &str)>) -> Response {
+ let mut error = json!({
+ "root_cause": [{ "type": error_type, "reason": reason }],
+ "type": error_type,
+ "reason": reason,
+ });
+ if let Some((cause_type, cause_reason)) = caused_by {
+ error["caused_by"] = json!({ "type": cause_type, "reason": cause_reason });
+ }
+ (
+ StatusCode::BAD_REQUEST,
+ Json(json!({ "error": error, "status": 400 })),
+ )
+ .into_response()
+}
+
+/// ES refuses a mapping that declares a field with an empty name
+/// (`mapper_parsing_exception` caused by `illegal_argument_exception`
+/// `field name cannot be an empty string`); XERJ accepted it. Checked through
+/// `properties` and multi-field `fields` at every depth.
+fn empty_field_name_refusal(props: &serde_json::Map<String, Value>) -> Option<Response> {
+ fn has_empty(props: &serde_json::Map<String, Value>) -> bool {
+ props.iter().any(|(name, spec)| {
+ name.is_empty()
+ || ["properties", "fields"].iter().any(|k| {
+ spec.get(*k)
+ .and_then(Value::as_object)
+ .is_some_and(has_empty)
+ })
+ })
+ }
+ has_empty(props).then(|| {
+ es_bad_request(
+ "mapper_parsing_exception",
+ "Failed to parse mapping: field name cannot be an empty string",
+ Some((
+ "illegal_argument_exception",
+ "field name cannot be an empty string",
+ )),
+ )
+ })
+}
+
+/// One index setting from a create-index `settings` object, in any of the
+/// spellings ES accepts: `{"index.a.b": v}`, `{"a.b": v}`,
+/// `{"index": {"a": {"b": v}}}`, `{"a": {"b": v}}` or `{"index": {"a.b": v}}`.
+fn create_setting<'a>(settings: &'a Value, path: &[&str]) -> Option<&'a Value> {
+ let dotted = path.join(".");
+ let nested = |root: &'a Value| {
+ path.iter()
+ .try_fold(root, |v, key| v.get(*key))
+ .or_else(|| root.get(&dotted))
+ };
+ settings
+ .get(format!("index.{dotted}"))
+ .or_else(|| settings.get("index").and_then(nested))
+ .or_else(|| nested(settings))
+}
+
+/// Request-shape and setting checks ES applies to a create-index body before
+/// anything is created (`indices.create/10_basic.yml`,
+/// `search/550_ignore_above_invalid.yml`). Each was previously accepted with
+/// a 200. Run before the index exists, so a refusal leaves nothing behind.
+fn create_index_body_refusal(index: &str, body: &Value) -> Option<Response> {
+ for section in ["mappings", "settings", "aliases"] {
+ if let Some(v) = body.get(section) {
+ if !v.is_object() && !v.is_null() {
+ return Some(es_bad_request(
+ "illegal_argument_exception",
+ &format!(
+ "Malformed [{section}] section for index [{index}], should include an \
+ inner object describing the {section}"
+ ),
+ None,
+ ));
+ }
+ }
+ }
+ if let Some(props) = body.pointer("/mappings/properties") {
+ if !props.is_object() {
+ return Some(es_bad_request(
+ "mapper_parsing_exception",
+ "Failed to parse mapping: [properties] must be a map type",
+ None,
+ ));
+ }
+ }
+ if let Some(settings) = body.get("settings") {
+ if create_setting(settings, &["soft_deletes", "enabled"])
+ .is_some_and(|v| v == &json!(false) || v == &json!("false"))
+ {
+ return Some(es_bad_request(
+ "illegal_argument_exception",
+ "Creating indices with soft-deletes disabled is no longer supported. Please do \
+ not specify a value for setting [index.soft_deletes.enabled].",
+ None,
+ ));
+ }
+ if create_setting(settings, &["mode"]).and_then(Value::as_str) == Some("lookup") {
+ let shards = create_setting(settings, &["number_of_shards"]).and_then(|v| {
+ v.as_u64()
+ .or_else(|| v.as_str().and_then(|s| s.parse().ok()))
+ });
+ if let Some(n) = shards.filter(|n| *n != 1) {
+ return Some(es_bad_request(
+ "illegal_argument_exception",
+ &format!(
+ "index with [lookup] mode must have [index.number_of_shards] set to 1 \
+ or unset; provided {n}"
+ ),
+ None,
+ ));
+ }
+ }
+ let ignore_above = create_setting(settings, &["mapping", "ignore_above"]).and_then(|v| {
+ v.as_i64()
+ .or_else(|| v.as_str().and_then(|s| s.parse().ok()))
+ });
+ if let Some(n) = ignore_above.filter(|n| *n < 0) {
+ return Some(es_bad_request(
+ "illegal_argument_exception",
+ &format!(
+ "Failed to parse value [{n}] for setting [index.mapping.ignore_above] must be \
+ >= 0"
+ ),
+ None,
+ ));
+ }
+ }
+ if let Some(aliases) = body.get("aliases").and_then(Value::as_object) {
+ for (alias, spec) in aliases {
+ if let Some(reason) = alias_value_type_error(alias, spec) {
+ return Some(es_bad_request("illegal_argument_exception", &reason, None));
+ }
+ }
+ }
+ None
+}
+
+/// The Java type name ES's alias parser reports for a JSON value.
+fn alias_value_type_name(v: &Value) -> &'static str {
+ match v {
+ Value::String(_) => "String",
+ Value::Bool(_) => "boolean",
+ Value::Number(_) => "Number",
+ Value::Array(_) => "Array",
+ Value::Object(_) => "Object",
+ Value::Null => "null",
+ }
+}
+
+/// ES's `Unsupported <Type> type value [v] for field [f] in alias [a]`: the
+/// flag fields take booleans, the routing fields strings or numbers.
+fn alias_value_type_error(alias: &str, spec: &Value) -> Option<String> {
+ let spec = spec.as_object()?;
+ for (field, v) in spec {
+ let ok = match field.as_str() {
+ "is_write_index" | "is_hidden" => v.is_boolean() || v.is_null(),
+ "routing" | "index_routing" | "search_routing" => {
+ v.is_string() || v.is_number() || v.is_null()
+ }
+ _ => true,
+ };
+ if !ok {
+ let shown = match v {
+ Value::String(s) => s.clone(),
+ other => other.to_string(),
+ };
+ return Some(format!(
+ "Unsupported {} type value [{shown}] for field [{field}] in alias [{alias}]",
+ alias_value_type_name(v)
+ ));
+ }
+ }
+ None
+}
+
/// ES mapper_parsing_exception emits for unsupported field types. Accept
/// the set of types we actually implement and explicitly reject the few
/// that YAML tests exercise as "invalid" (`baz`, `foobar`, etc.).
@@ -1272,7 +1458,36 @@ fn validate_properties(props: &serde_json::Map<String, Value>) -> Result<(), Str
// as a multi-field would otherwise still be accepted, echoed back
// by `GET /_mapping`, and dropped on the floor. One shared
// resolver means one accepted set and one error message.
+ if matches!(t, "keyword" | "flattened" | "wildcard") {
+ if let Some(n) = obj.get("ignore_above").and_then(Value::as_i64) {
+ if n < 0 {
+ return Err(format!(
+ "Failed to parse mapping: [ignore_above] must be positive, got [{n}]"
+ ));
+ }
+ }
+ }
if t == "dense_vector" {
+ if let Some(opts) = obj.get("index_options") {
+ if obj.get("index").and_then(Value::as_bool) == Some(false) {
+ return Err(
+ "Failed to parse mapping: Field [index_options] requires field \
+ [index] to be configured and set to true"
+ .to_string(),
+ );
+ }
+ let element = obj
+ .get("element_type")
+ .and_then(Value::as_str)
+ .unwrap_or("float");
+ let kind = opts.get("type").and_then(Value::as_str).unwrap_or("");
+ if kind.starts_with("bbq") && matches!(element, "byte" | "bit") {
+ return Err(format!(
+ "Failed to parse mapping: [index_options] of type [{kind}] is not \
+ supported for element_type [{element}]"
+ ));
+ }
+ }
resolve_vector_quantization(name, spec)?;
}
}
@@ -1883,6 +2098,20 @@ async fn get_index_inner(
index: String,
params: GetIndexParams,
) -> axum::response::Response {
+ // A name that starts with `_` (other than `_all`) can never be an index:
+ // ES answers 400 `invalid_index_name_exception`, not a 404. (A leading
+ // `-` in a list is an exclusion, so only `_` is checked.)
+ if let Some(bad) = index
+ .split(',')
+ .map(str::trim)
+ .find(|p| p.starts_with('_') && *p != "_all")
+ {
+ return es_bad_request(
+ "invalid_index_name_exception",
+ &format!("Invalid index name [{bad}], must not start with '_', '-', or '+'"),
+ None,
+ );
+ }
let ignore_unavailable = params.ignore_unavailable.as_deref() == Some("true");
// ES default: wildcard / `_all` selectors default `allow_no_indices=true`
// (a pattern that matches nothing returns `{}` rather than 404). Explicit
@@ -2288,6 +2517,9 @@ pub async fn put_mapping(
// Validate any `properties` field types once, up front.
if let Some(props) = body.get("properties").and_then(Value::as_object) {
+ if let Some(refusal) = empty_field_name_refusal(props) {
+ return refusal;
+ }
if let Err(reason) = validate_properties(props) {
let e = xerj_common::XerjError::invalid_mapping(reason);
return ApiError::new(e).into_response();
@@ -22673,6 +22905,67 @@ pub async fn put_alias(
Path((index, alias)): Path<(String, String)>,
body: OptionalJson<Value>,
) -> impl IntoResponse {
+ // ES's alias-name and body checks (`indices.put_alias/10_basic.yml`),
+ // before anything is attached.
+ const ALIAS_FORBIDDEN: &[char] = &[' ', '"', '*', '\\', '<', '|', ',', '>', '/', '?'];
+ if alias.contains(ALIAS_FORBIDDEN) {
+ return es_bad_request(
+ "invalid_alias_name_exception",
+ &format!(
+ "Invalid alias name [{alias}]: must not contain the following characters \
+ [ , \", *, \\, <, |, ,, >, /, ?]"
+ ),
+ None,
+ );
+ }
+ // A concrete index of that name (not this alias already existing —
+ // re-putting an alias to change its filter or routing stays allowed).
+ if !state.engine.aliases.contains_key(&alias) && state.engine.get_index(&alias).is_ok() {
+ return es_bad_request(
+ "invalid_alias_name_exception",
+ &format!(
+ "Invalid alias name [{alias}]: an index or data stream exists with the same name \
+ as the alias"
+ ),
+ None,
+ );
+ }
+ if let Some(fields) = body.0.as_ref().and_then(Value::as_object) {
+ const ALIAS_FIELDS: &[&str] = &[
+ "filter",
+ "routing",
+ "index_routing",
+ "search_routing",
+ "is_write_index",
+ "is_hidden",
+ ];
+ for (field, v) in fields {
+ if !ALIAS_FIELDS.contains(&field.as_str()) {
+ return es_bad_request(
+ "illegal_argument_exception",
+ &format!("[{alias}] Unsupported field [{field}]"),
+ None,
+ );
+ }
+ if field.ends_with("routing") && (v.is_array() || v.is_object()) {
+ let token = if v.is_array() {
+ "START_ARRAY"
+ } else {
+ "START_OBJECT"
+ };
+ return es_bad_request(
+ "illegal_argument_exception",
+ &format!(
+ "[{alias}] failed to parse field [{field}]: Unexpected token [{token}]"
+ ),
+ None,
+ );
+ }
+ }
+ if let Some(reason) = alias_value_type_error(&alias, &Value::Object(fields.clone())) {
+ return es_bad_request("illegal_argument_exception", &reason, None);
+ }
+ }
let targets = resolve_index_selector(&state, &index).await;
let alias_meta: Value = body
.0
@@ -44011,6 +44304,18 @@ pub async fn put_legacy_template(
Path(name): Path<String>,
Json(mut body): Json<Value>,
) -> impl IntoResponse {
+ let has_patterns = match body.get("index_patterns") {
+ Some(Value::String(s)) => !s.is_empty(),
+ Some(Value::Array(a)) => !a.is_empty(),
+ _ => false,
+ };
+ if !has_patterns {
+ return es_bad_request(
+ "action_request_validation_exception",
+ "Validation Failed: 1: index patterns are missing;",
+ None,
+ );
+ }
if let Some(ip) = body.get("index_patterns") {
if ip.is_string() {
body["index_patterns"] = json!([ip.as_str().unwrap_or_default()]);
diff --git a/engine/tests/es-compat-yaml/catch-lenient.txt b/engine/tests/es-compat-yaml/catch-lenient.txt
index 080b5f5d2..6e774834f 100644
--- a/engine/tests/es-compat-yaml/catch-lenient.txt
+++ b/engine/tests/es-compat-yaml/catch-lenient.txt
@@ -101,17 +101,6 @@ aggregations/top_hits.yml · having
aggregations/weighted_avg.yml · Multi weight field not allowed
bulk/10_basic.yml · empty action
bulk/60_deprecated.yml · Deprecated parameters should fail in Bulk query
-indices/indices.create/10_basic.yml · Create index with invalid mappings
-indices/indices.create/10_basic.yml · Create index without soft deletes
-indices/indices.create/10_basic.yml · Create lookup index with two shards
-indices/indices.create/10_basic.yml · Poorly formatted request returns bad_request
-indices/indices.create/10_basic.yml · Throw exception for unsupported value type
-indices/indices.get/10_basic.yml · Should return an exception when querying invalid indices
-indices/indices.put_alias/10_basic.yml · Can't create alias with invalid characters
-indices/indices.put_alias/10_basic.yml · Can't create alias with the same name as an index
-indices/indices.put_alias/10_basic.yml · Throw exception for unknown token
-indices/indices.put_mapping/10_basic.yml · Create index with invalid mappings
-indices/indices.put_template/10_basic.yml · Put index template without index_patterns
indices/indices.put_template/10_basic.yml · Put template create
scroll/10_basic.yml · Scroll cannot used the request cache
scroll/10_basic.yml · Scroll with size 0
@@ -152,10 +141,4 @@ search/40_indices_boost.yml · Indices boost for nonexistent index/alias
search/40_indices_boost.yml · Remote expressions not supported in indices boost
search/500_date_range.yml · test bad dates in range - future
search/500_date_range.yml · test bad dates in range - past
-search/550_ignore_above_invalid.yml · flattened ignore_above mapping setting negative value
-search/550_ignore_above_invalid.yml · ignore_above index setting negative value
-search/550_ignore_above_invalid.yml · keyword ignore_above mapping setting negative value
-search/550_ignore_above_invalid.yml · wildcard ignore_above mapping setting negative value
search/90_search_after.yml · Format sort values
-vectors/search.vectors/46_knn_search_bbq_ivf.yml · Test bad quantization parameters
-vectors/search.vectors/search.vectors/46_knn_search_bbq_ivf.yml · Test bad quantization parameters