/* * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /* * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ package org.opensearch.index.mapper; import org.opensearch.common.Explicit; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.DynamicTemplate.XContentFieldType; import org.opensearch.index.mapper.MapperService.MergeReason; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import static org.opensearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.opensearch.index.mapper.TypeParsers.parseDateTimeFormatter; /** * The root object mapper for a document * * @opensearch.internal */ public class RootObjectMapper extends ObjectMapper { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RootObjectMapper.class); /** * Default parameters for root object * * @opensearch.internal */ public static class Defaults { public static final DateFormatter[] DYNAMIC_DATE_TIME_FORMATTERS = new DateFormatter[] { DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFormatter.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis") }; public static final boolean DATE_DETECTION = true; public static final boolean NUMERIC_DETECTION = false; } /** * Builder for the root object * * @opensearch.internal */ public static class Builder extends ObjectMapper.Builder { protected Explicit dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false); protected Explicit dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false); protected Explicit dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false); protected Explicit numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false); public Builder(String name) { super(name); this.builder = this; } public Builder dynamicDateTimeFormatter(Collection dateTimeFormatters) { this.dynamicDateTimeFormatters = new Explicit<>(dateTimeFormatters.toArray(new DateFormatter[0]), true); return this; } public Builder dynamicTemplates(Collection templates) { this.dynamicTemplates = new Explicit<>(templates.toArray(new DynamicTemplate[0]), true); return this; } @Override public RootObjectMapper build(BuilderContext context) { return (RootObjectMapper) super.build(context); } @Override protected ObjectMapper createMapper( String name, String fullPath, Explicit enabled, Nested nested, Dynamic dynamic, Map mappers, @Nullable Settings settings ) { assert !nested.isNested(); return new RootObjectMapper( name, enabled, dynamic, mappers, dynamicDateTimeFormatters, dynamicTemplates, dateDetection, numericDetection, settings ); } } /** * Removes redundant root includes in {@link ObjectMapper.Nested} trees to avoid duplicate * fields on the root mapper when {@code isIncludeInRoot} is {@code true} for a node that is * itself included into a parent node, for which either {@code isIncludeInRoot} is * {@code true} or which is transitively included in root by a chain of nodes with * {@code isIncludeInParent} returning {@code true}. */ public void fixRedundantIncludes() { fixRedundantIncludes(this, true); } private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean parentIncluded) { for (Mapper mapper : objectMapper) { if (mapper instanceof ObjectMapper) { ObjectMapper child = (ObjectMapper) mapper; Nested nested = child.nested(); boolean isNested = nested.isNested(); boolean includeInRootViaParent = parentIncluded && isNested && nested.isIncludeInParent(); boolean includedInRoot = isNested && nested.isIncludeInRoot(); if (includeInRootViaParent && includedInRoot) { nested.setIncludeInParent(true); nested.setIncludeInRoot(false); } fixRedundantIncludes(child, includeInRootViaParent || includedInRoot); } } } /** * Type parser for the root object * * @opensearch.internal */ public static class TypeParser extends ObjectMapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { RootObjectMapper.Builder builder = new Builder(name); Iterator> iterator = node.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); String fieldName = entry.getKey(); Object fieldNode = entry.getValue(); if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder) || processField(builder, fieldName, fieldNode, parserContext)) { iterator.remove(); } } return builder; } protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) { if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { if (fieldNode instanceof List) { List formatters = new ArrayList<>(); for (Object formatter : (List) fieldNode) { if (formatter.toString().startsWith("epoch_")) { throw new MapperParsingException("Epoch [" + formatter + "] is not supported as dynamic date format"); } formatters.add(parseDateTimeFormatter(formatter)); } builder.dynamicDateTimeFormatter(formatters); } else if ("none".equals(fieldNode.toString())) { builder.dynamicDateTimeFormatter(Collections.emptyList()); } else { builder.dynamicDateTimeFormatter(Collections.singleton(parseDateTimeFormatter(fieldNode))); } return true; } else if (fieldName.equals("dynamic_templates")) { // "dynamic_templates" : [ // { // "template_1" : { // "match" : "*_test", // "match_mapping_type" : "string", // "mapping" : { "type" : "keyword", "store" : "yes" } // } // } // ] if ((fieldNode instanceof List) == false) { throw new MapperParsingException("Dynamic template syntax error. An array of named objects is expected."); } List tmplNodes = (List) fieldNode; List templates = new ArrayList<>(); for (Object tmplNode : tmplNodes) { Map tmpl = (Map) tmplNode; if (tmpl.size() != 1) { throw new MapperParsingException("A dynamic template must be defined with a name"); } Map.Entry entry = tmpl.entrySet().iterator().next(); String templateName = entry.getKey(); Map templateParams = (Map) entry.getValue(); DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams); if (template != null) { validateDynamicTemplate(parserContext, template); templates.add(template); } } builder.dynamicTemplates(templates); return true; } else if (fieldName.equals("date_detection")) { builder.dateDetection = new Explicit<>(nodeBooleanValue(fieldNode, "date_detection"), true); return true; } else if (fieldName.equals("numeric_detection")) { builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true); return true; } return false; } } private Explicit dynamicDateTimeFormatters; private Explicit dateDetection; private Explicit numericDetection; private Explicit dynamicTemplates; RootObjectMapper( String name, Explicit enabled, Dynamic dynamic, Map mappers, Explicit dynamicDateTimeFormatters, Explicit dynamicTemplates, Explicit dateDetection, Explicit numericDetection, Settings settings ) { super(name, name, enabled, Nested.NO, dynamic, mappers, settings); this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; this.numericDetection = numericDetection; } @Override public ObjectMapper mappingUpdate(Mapper mapper) { RootObjectMapper update = (RootObjectMapper) super.mappingUpdate(mapper); // for dynamic updates, no need to carry root-specific options, we just // set everything to they implicit default value so that they are not // applied at merge time update.dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false); update.dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false); update.dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false); update.numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false); return update; } public boolean dateDetection() { return this.dateDetection.value(); } public boolean numericDetection() { return this.numericDetection.value(); } public DateFormatter[] dynamicDateTimeFormatters() { return dynamicDateTimeFormatters.value(); } public DynamicTemplate[] dynamicTemplates() { return dynamicTemplates.value(); } @SuppressWarnings("rawtypes") public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) { return findTemplateBuilder(context, name, matchType, null); } public Mapper.Builder findTemplateBuilder(ParseContext context, String name, DateFormatter dateFormatter) { return findTemplateBuilder(context, name, XContentFieldType.DATE, dateFormatter); } /** * Find a template. Returns {@code null} if no template could be found. * @param name the field name * @param matchType the type of the field in the json document or null if unknown * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format * @return a mapper builder, or null if there is no template for such a field */ @SuppressWarnings("rawtypes") private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); if (dynamicTemplate == null) { return null; } String dynamicType = matchType.defaultMappingType(); Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext(dateFormat); String mappingType = dynamicTemplate.mappingType(dynamicType); Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); if (typeParser == null) { throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + name + "]"); } return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext); } public DynamicTemplate findTemplate(ContentPath path, String name, XContentFieldType matchType) { final String pathAsString = path.pathAsText(name); for (DynamicTemplate dynamicTemplate : dynamicTemplates.value()) { if (dynamicTemplate.match(pathAsString, name, matchType)) { return dynamicTemplate; } } return null; } @Override public RootObjectMapper merge(Mapper mergeWith, MergeReason reason) { return (RootObjectMapper) super.merge(mergeWith, reason); } @Override protected void doMerge(ObjectMapper mergeWith, MergeReason reason) { super.doMerge(mergeWith, reason); RootObjectMapper mergeWithObject = (RootObjectMapper) mergeWith; if (mergeWithObject.numericDetection.explicit()) { this.numericDetection = mergeWithObject.numericDetection; } if (mergeWithObject.dateDetection.explicit()) { this.dateDetection = mergeWithObject.dateDetection; } if (mergeWithObject.dynamicDateTimeFormatters.explicit()) { this.dynamicDateTimeFormatters = mergeWithObject.dynamicDateTimeFormatters; } if (mergeWithObject.dynamicTemplates.explicit()) { if (reason == MergeReason.INDEX_TEMPLATE) { Map templatesByKey = new LinkedHashMap<>(); for (DynamicTemplate template : this.dynamicTemplates.value()) { templatesByKey.put(template.name(), template); } for (DynamicTemplate template : mergeWithObject.dynamicTemplates.value()) { templatesByKey.put(template.name(), template); } DynamicTemplate[] mergedTemplates = templatesByKey.values().toArray(new DynamicTemplate[0]); this.dynamicTemplates = new Explicit<>(mergedTemplates, true); } else { this.dynamicTemplates = mergeWithObject.dynamicTemplates; } } } @Override protected void doXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { final boolean includeDefaults = params.paramAsBoolean("include_defaults", false); if (dynamicDateTimeFormatters.explicit() || includeDefaults) { builder.startArray("dynamic_date_formats"); for (DateFormatter dateTimeFormatter : dynamicDateTimeFormatters.value()) { builder.value(dateTimeFormatter.pattern()); } builder.endArray(); } if (dynamicTemplates.explicit() || includeDefaults) { builder.startArray("dynamic_templates"); for (DynamicTemplate dynamicTemplate : dynamicTemplates.value()) { builder.startObject(); builder.field(dynamicTemplate.name(), dynamicTemplate); builder.endObject(); } builder.endArray(); } if (dateDetection.explicit() || includeDefaults) { builder.field("date_detection", dateDetection.value()); } if (numericDetection.explicit() || includeDefaults) { builder.field("numeric_detection", numericDetection.value()); } } private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext, DynamicTemplate dynamicTemplate) { if (containsSnippet(dynamicTemplate.getMapping(), "{name}")) { // Can't validate template, because field names can't be guessed up front. return; } final XContentFieldType[] types; if (dynamicTemplate.getXContentFieldType() != null) { types = new XContentFieldType[] { dynamicTemplate.getXContentFieldType() }; } else { types = XContentFieldType.values(); } Exception lastError = null; boolean dynamicTemplateInvalid = true; for (XContentFieldType contentFieldType : types) { String defaultDynamicType = contentFieldType.defaultMappingType(); String mappingType = dynamicTemplate.mappingType(defaultDynamicType); Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); if (typeParser == null) { lastError = new IllegalArgumentException("No mapper found for type [" + mappingType + "]"); continue; } String templateName = "__dynamic__" + dynamicTemplate.name(); Map fieldTypeConfig = dynamicTemplate.mappingForName(templateName, defaultDynamicType); try { Mapper.Builder dummyBuilder = typeParser.parse(templateName, fieldTypeConfig, parserContext); fieldTypeConfig.remove("type"); if (fieldTypeConfig.isEmpty()) { Settings indexSettings = parserContext.mapperService().getIndexSettings().getSettings(); BuilderContext builderContext = new BuilderContext(indexSettings, new ContentPath(1)); dummyBuilder.build(builderContext); dynamicTemplateInvalid = false; break; } else { lastError = new IllegalArgumentException("Unused mapping attributes [" + fieldTypeConfig + "]"); } } catch (Exception e) { lastError = e; } } if (dynamicTemplateInvalid) { String message = String.format( Locale.ROOT, "dynamic template [%s] has invalid content [%s]", dynamicTemplate.getName(), Strings.toString(XContentType.JSON, dynamicTemplate) ); final String deprecationMessage; if (lastError != null) { deprecationMessage = String.format(Locale.ROOT, "%s, caused by [%s]", message, lastError.getMessage()); } else { deprecationMessage = message; } DEPRECATION_LOGGER.deprecate("invalid_dynamic_template_" + dynamicTemplate.getName(), deprecationMessage); } } private static boolean containsSnippet(Map map, String snippet) { for (Map.Entry entry : map.entrySet()) { String key = entry.getKey().toString(); if (key.contains(snippet)) { return true; } Object value = entry.getValue(); if (value instanceof Map) { if (containsSnippet((Map) value, snippet)) { return true; } } else if (value instanceof List) { if (containsSnippet((List) value, snippet)) { return true; } } else if (value instanceof String) { String valueString = (String) value; if (valueString.contains(snippet)) { return true; } } } return false; } private static boolean containsSnippet(List list, String snippet) { for (Object value : list) { if (value instanceof Map) { if (containsSnippet((Map) value, snippet)) { return true; } } else if (value instanceof List) { if (containsSnippet((List) value, snippet)) { return true; } } else if (value instanceof String) { String valueString = (String) value; if (valueString.contains(snippet)) { return true; } } } return false; } }