/* * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /* * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ package org.opensearch.cluster.metadata; import org.opensearch.OpenSearchParseException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.ClusterState; import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.regex.Regex; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateMathParser; import org.opensearch.common.time.DateUtils; import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.Strings; import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.indices.IndexClosedException; import org.opensearch.indices.InvalidIndexNameException; import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.SortedMap; import java.util.Spliterators; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; /** * Resolves index name from an expression * * @opensearch.internal */ public class IndexNameExpressionResolver { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexNameExpressionResolver.class); public static final String EXCLUDED_DATA_STREAMS_KEY = "opensearch.excluded_ds"; public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed"; private final DateMathExpressionResolver dateMathExpressionResolver = new DateMathExpressionResolver(); private final WildcardExpressionResolver wildcardExpressionResolver = new WildcardExpressionResolver(); private final List expressionResolvers = List.of(dateMathExpressionResolver, wildcardExpressionResolver); private final ThreadContext threadContext; public IndexNameExpressionResolver(ThreadContext threadContext) { this.threadContext = Objects.requireNonNull(threadContext, "Thread Context must not be null"); } /** * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options * are encapsulated in the specified request. */ public String[] concreteIndexNames(ClusterState state, IndicesRequest request) { Context context = new Context( state, request.indicesOptions(), false, false, request.includeDataStreams(), isSystemIndexAccessAllowed() ); return concreteIndexNames(context, request.indices()); } /** * Same as {@link #concreteIndexNames(ClusterState, IndicesRequest)}, but access to system indices is always allowed. */ public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, IndicesRequest request) { Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams(), true); return concreteIndexNames(context, request.indices()); } /** * Same as {@link #concreteIndices(ClusterState, IndicesOptions, String...)}, but the index expressions and options * are encapsulated in the specified request and resolves data streams. */ public Index[] concreteIndices(ClusterState state, IndicesRequest request) { Context context = new Context( state, request.indicesOptions(), false, false, request.includeDataStreams(), isSystemIndexAccessAllowed() ); return concreteIndices(context, request.indices()); } /** * Translates the provided index expression into actual concrete indices, properly deduplicated. * * @param state the cluster state containing all the data to resolve to expressions to concrete indices * @param options defines how the aliases or indices need to be resolved to concrete indices * @param indexExpressions expressions that can be resolved to alias or index names. * @return the resolved concrete indices based on the cluster state, indices options and index expressions * @throws IndexNotFoundException if one of the index expressions is pointing to a missing index or alias and the * provided indices options in the context don't allow such a case, or if the final result of the indices resolution * contains no indices and the indices options in the context don't allow such a case. * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided * indices options in the context don't allow such a case. */ public String[] concreteIndexNames(ClusterState state, IndicesOptions options, String... indexExpressions) { Context context = new Context(state, options, isSystemIndexAccessAllowed()); return concreteIndexNames(context, indexExpressions); } public String[] concreteIndexNames(ClusterState state, IndicesOptions options, boolean includeDataStreams, String... indexExpressions) { Context context = new Context(state, options, false, false, includeDataStreams, isSystemIndexAccessAllowed()); return concreteIndexNames(context, indexExpressions); } public String[] concreteIndexNames(ClusterState state, IndicesOptions options, IndicesRequest request) { Context context = new Context(state, options, false, false, request.includeDataStreams(), isSystemIndexAccessAllowed()); return concreteIndexNames(context, request.indices()); } public List dataStreamNames(ClusterState state, IndicesOptions options, String... indexExpressions) { // Allow system index access - they'll be filtered out below as there's no such thing (yet) as system data streams Context context = new Context(state, options, false, false, true, true, true); if (indexExpressions == null || indexExpressions.length == 0) { indexExpressions = new String[] { "*" }; } List dataStreams = wildcardExpressionResolver.resolve(context, Arrays.asList(indexExpressions)); return ((dataStreams == null) ? List.of() : dataStreams).stream() .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) .filter(ia -> ia.getType() == IndexAbstraction.Type.DATA_STREAM) .map(IndexAbstraction::getName) .collect(Collectors.toList()); } /** * Translates the provided index expression into actual concrete indices, properly deduplicated. * * @param state the cluster state containing all the data to resolve to expressions to concrete indices * @param options defines how the aliases or indices need to be resolved to concrete indices * @param indexExpressions expressions that can be resolved to alias or index names. * @return the resolved concrete indices based on the cluster state, indices options and index expressions * @throws IndexNotFoundException if one of the index expressions is pointing to a missing index or alias and the * provided indices options in the context don't allow such a case, or if the final result of the indices resolution * contains no indices and the indices options in the context don't allow such a case. * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided * indices options in the context don't allow such a case. */ public Index[] concreteIndices(ClusterState state, IndicesOptions options, String... indexExpressions) { return concreteIndices(state, options, false, indexExpressions); } public Index[] concreteIndices(ClusterState state, IndicesOptions options, boolean includeDataStreams, String... indexExpressions) { Context context = new Context(state, options, false, false, includeDataStreams, isSystemIndexAccessAllowed()); return concreteIndices(context, indexExpressions); } /** * Translates the provided index expression into actual concrete indices, properly deduplicated. * * @param state the cluster state containing all the data to resolve to expressions to concrete indices * @param startTime The start of the request where concrete indices is being invoked for * @param request request containing expressions that can be resolved to alias, index, or data stream names. * @return the resolved concrete indices based on the cluster state, indices options and index expressions * provided indices options in the context don't allow such a case, or if the final result of the indices resolution * contains no indices and the indices options in the context don't allow such a case. * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided * indices options in the context don't allow such a case. */ public Index[] concreteIndices(ClusterState state, IndicesRequest request, long startTime) { Context context = new Context( state, request.indicesOptions(), startTime, false, false, request.includeDataStreams(), false, isSystemIndexAccessAllowed() ); return concreteIndices(context, request.indices()); } String[] concreteIndexNames(Context context, String... indexExpressions) { Index[] indexes = concreteIndices(context, indexExpressions); String[] names = new String[indexes.length]; for (int i = 0; i < indexes.length; i++) { names[i] = indexes[i].getName(); } return names; } Index[] concreteIndices(Context context, String... indexExpressions) { if (indexExpressions == null || indexExpressions.length == 0) { indexExpressions = new String[] { Metadata.ALL }; } Metadata metadata = context.getState().metadata(); IndicesOptions options = context.getOptions(); // If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices // option. At some point we should change this, because there shouldn't be a reason why whether a single index // or multiple indices are specified yield different behaviour. final boolean failNoIndices = indexExpressions.length == 1 ? !options.allowNoIndices() : !options.ignoreUnavailable(); List expressions = Arrays.asList(indexExpressions); for (ExpressionResolver expressionResolver : expressionResolvers) { expressions = expressionResolver.resolve(context, expressions); } if (expressions.isEmpty()) { if (!options.allowNoIndices()) { IndexNotFoundException infe; if (indexExpressions.length == 1) { if (indexExpressions[0].equals(Metadata.ALL)) { infe = new IndexNotFoundException("no indices exist", (String) null); } else { infe = new IndexNotFoundException((String) null); } } else { infe = new IndexNotFoundException((String) null); } infe.setResources("index_expression", indexExpressions); throw infe; } else { return Index.EMPTY_ARRAY; } } boolean excludedDataStreams = false; final Set concreteIndices = new HashSet<>(expressions.size()); for (String expression : expressions) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(expression); if (indexAbstraction == null) { if (failNoIndices) { IndexNotFoundException infe; if (expression.equals(Metadata.ALL)) { infe = new IndexNotFoundException("no indices exist", expression); } else { infe = new IndexNotFoundException(expression); } infe.setResources("index_expression", expression); throw infe; } else { continue; } } else if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.getOptions().ignoreAliases()) { if (failNoIndices) { throw aliasesNotSupportedException(expression); } else { continue; } } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.includeDataStreams() == false) { excludedDataStreams = true; continue; } if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.isResolveToWriteIndex()) { IndexMetadata writeIndex = indexAbstraction.getWriteIndex(); if (writeIndex == null) { throw new IllegalArgumentException( "no write index is defined for alias [" + indexAbstraction.getName() + "]." + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + " indices without one being designated as a write index" ); } if (addIndex(writeIndex, context)) { concreteIndices.add(writeIndex.getIndex()); } } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.isResolveToWriteIndex()) { IndexMetadata writeIndex = indexAbstraction.getWriteIndex(); if (addIndex(writeIndex, context)) { concreteIndices.add(writeIndex.getIndex()); } } else { if (indexAbstraction.getIndices().size() > 1 && !options.allowAliasesToMultipleIndices()) { String[] indexNames = new String[indexAbstraction.getIndices().size()]; int i = 0; for (IndexMetadata indexMetadata : indexAbstraction.getIndices()) { indexNames[i++] = indexMetadata.getIndex().getName(); } throw new IllegalArgumentException( indexAbstraction.getType().getDisplayName() + " [" + expression + "] has more than one index associated with it " + Arrays.toString(indexNames) + ", can't execute a single index op" ); } for (IndexMetadata index : indexAbstraction.getIndices()) { if (shouldTrackConcreteIndex(context, options, index)) { concreteIndices.add(index.getIndex()); } } } } if (options.allowNoIndices() == false && concreteIndices.isEmpty()) { IndexNotFoundException infe = new IndexNotFoundException((String) null); infe.setResources("index_expression", indexExpressions); if (excludedDataStreams) { // Allows callers to handle IndexNotFoundException differently based on whether data streams were excluded. infe.addMetadata(EXCLUDED_DATA_STREAMS_KEY, "true"); } throw infe; } checkSystemIndexAccess(context, metadata, concreteIndices, indexExpressions); return concreteIndices.toArray(new Index[0]); } private void checkSystemIndexAccess(Context context, Metadata metadata, Set concreteIndices, String[] originalPatterns) { if (context.isSystemIndexAccessAllowed() == false) { final List resolvedSystemIndices = concreteIndices.stream() .map(metadata::index) .filter(IndexMetadata::isSystem) .map(i -> i.getIndex().getName()) .sorted() // reliable order for testing .collect(Collectors.toList()); if (resolvedSystemIndices.isEmpty() == false) { resolvedSystemIndices.forEach( systemIndexName -> deprecationLogger.deprecate( "open_system_index_access_" + systemIndexName, "this request accesses system indices: [{}], but in a future major version, direct access to system " + "indices will be prevented by default", systemIndexName ) ); } } } private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, IndexMetadata index) { if (index.getState() == IndexMetadata.State.CLOSE) { if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) { throw new IndexClosedException(index.getIndex()); } else { return options.forbidClosedIndices() == false && addIndex(index, context); } } else if (index.getState() == IndexMetadata.State.OPEN) { return addIndex(index, context); } else { throw new IllegalStateException("index state [" + index.getState() + "] not supported"); } } private static boolean addIndex(IndexMetadata metadata, Context context) { // This used to check the `index.search.throttled` setting, but we eventually decided that it was // trappy to hide throttled indices by default. In order to avoid breaking backward compatibility, // we changed it to look at the `index.frozen` setting instead, since frozen indices were the only // type of index to use the `search_throttled` threadpool at that time. // NOTE: The Setting object was defined in an external plugin prior to OpenSearch fork. return (context.options.ignoreThrottled() && metadata.getSettings().getAsBoolean("index.frozen", false)) == false; } private static IllegalArgumentException aliasesNotSupportedException(String expression) { return new IllegalArgumentException( "The provided expression [" + expression + "] matches an " + "alias, specify the corresponding concrete indices instead." ); } /** * Utility method that allows to resolve an index expression to its corresponding single concrete index. * Callers should make sure they provide proper {@link org.opensearch.action.support.IndicesOptions} * that require a single index as a result. The indices resolution must in fact return a single index when * using this method, an {@link IllegalArgumentException} gets thrown otherwise. * * @param state the cluster state containing all the data to resolve to expression to a concrete index * @param request The request that defines how the an alias or an index need to be resolved to a concrete index * and the expression that can be resolved to an alias or an index name. * @throws IllegalArgumentException if the index resolution lead to more than one index * @return the concrete index obtained as a result of the index resolution */ public Index concreteSingleIndex(ClusterState state, IndicesRequest request) { String indexExpression = CollectionUtils.isEmpty(request.indices()) ? null : request.indices()[0]; Index[] indices = concreteIndices(state, request.indicesOptions(), indexExpression); if (indices.length != 1) { throw new IllegalArgumentException( "unable to return a single index as the index and options" + " provided got resolved to multiple indices" ); } return indices[0]; } /** * Utility method that allows to resolve an index expression to its corresponding single write index. * * @param state the cluster state containing all the data to resolve to expression to a concrete index * @param request The request that defines how the an alias or an index need to be resolved to a concrete index * and the expression that can be resolved to an alias or an index name. * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index * @return the write index obtained as a result of the index resolution */ public Index concreteWriteIndex(ClusterState state, IndicesRequest request) { if (request.indices() == null || (request.indices() != null && request.indices().length != 1)) { throw new IllegalArgumentException("indices request must specify a single index expression"); } return concreteWriteIndex(state, request.indicesOptions(), request.indices()[0], false, request.includeDataStreams()); } /** * Utility method that allows to resolve an index expression to its corresponding single write index. * * @param state the cluster state containing all the data to resolve to expression to a concrete index * @param options defines how the aliases or indices need to be resolved to concrete indices * @param index index that can be resolved to alias or index name. * @param allowNoIndices whether to allow resolve to no index * @param includeDataStreams Whether data streams should be included in the evaluation. * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index * @return the write index obtained as a result of the index resolution or null if no index */ public Index concreteWriteIndex( ClusterState state, IndicesOptions options, String index, boolean allowNoIndices, boolean includeDataStreams ) { IndicesOptions combinedOptions = IndicesOptions.fromOptions( options.ignoreUnavailable(), allowNoIndices, options.expandWildcardsOpen(), options.expandWildcardsClosed(), options.expandWildcardsHidden(), options.allowAliasesToMultipleIndices(), options.forbidClosedIndices(), options.ignoreAliases(), options.ignoreThrottled() ); Context context = new Context(state, combinedOptions, false, true, includeDataStreams, isSystemIndexAccessAllowed()); Index[] indices = concreteIndices(context, index); if (allowNoIndices && indices.length == 0) { return null; } if (indices.length != 1) { throw new IllegalArgumentException( "The index expression [" + index + "] and options provided did not point to a single write-index" ); } return indices[0]; } /** * @return whether the specified index, data stream or alias exists. * If the data stream, index or alias contains date math then that is resolved too. */ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, isSystemIndexAccessAllowed()); String resolvedAliasOrIndex = dateMathExpressionResolver.resolveExpression(indexAbstraction, context); return state.metadata().getIndicesLookup().containsKey(resolvedAliasOrIndex); } /** * @return If the specified string is data math expression then this method returns the resolved expression. */ public String resolveDateMathExpression(String dateExpression) { // The data math expression resolver doesn't rely on cluster state or indices options, because // it just resolves the date math to an actual date. return dateMathExpressionResolver.resolveExpression(dateExpression, new Context(null, null, isSystemIndexAccessAllowed())); } /** * Resolve an array of expressions to the set of indices and aliases that these expressions match. */ public Set resolveExpressions(ClusterState state, String... expressions) { Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false, true, isSystemIndexAccessAllowed()); List resolvedExpressions = Arrays.asList(expressions); for (ExpressionResolver expressionResolver : expressionResolvers) { resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions); } return Collections.unmodifiableSet(new HashSet<>(resolvedExpressions)); } /** * Iterates through the list of indices and selects the effective list of filtering aliases for the * given index. *

Only aliases with filters are returned. If the indices list contains a non-filtering reference to * the index itself - null is returned. Returns {@code null} if no filtering is required. * NOTE: The provided expressions must have been resolved already via {@link #resolveExpressions}. */ public String[] filteringAliases(ClusterState state, String index, Set resolvedExpressions) { return indexAliases(state, index, AliasMetadata::filteringRequired, false, resolvedExpressions); } /** * Whether to generate the candidate set from index aliases, or from the set of resolved expressions. * @param indexAliasesSize the number of aliases of the index * @param resolvedExpressionsSize the number of resolved expressions */ // pkg-private for testing boolean iterateIndexAliases(int indexAliasesSize, int resolvedExpressionsSize) { return indexAliasesSize <= resolvedExpressionsSize; } /** * Iterates through the list of indices and selects the effective list of required aliases for the given index. *

Only aliases where the given predicate tests successfully are returned. If the indices list contains a non-required reference to * the index itself - null is returned. Returns {@code null} if no filtering is required. *

NOTE: the provided expressions must have been resolved already via {@link #resolveExpressions}. */ public String[] indexAliases( ClusterState state, String index, Predicate requiredAlias, boolean skipIdentity, Set resolvedExpressions ) { if (isAllIndices(resolvedExpressions)) { return null; } final IndexMetadata indexMetadata = state.metadata().getIndices().get(index); if (indexMetadata == null) { // Shouldn't happen throw new IndexNotFoundException(index); } if (skipIdentity == false && resolvedExpressions.contains(index)) { return null; } final Map indexAliases = indexMetadata.getAliases(); final AliasMetadata[] aliasCandidates; if (iterateIndexAliases(indexAliases.size(), resolvedExpressions.size())) { // faster to iterate indexAliases aliasCandidates = StreamSupport.stream(Spliterators.spliteratorUnknownSize(indexAliases.values().iterator(), 0), false) .filter(aliasMetadata -> resolvedExpressions.contains(aliasMetadata.alias())) .toArray(AliasMetadata[]::new); } else { // faster to iterate resolvedExpressions aliasCandidates = resolvedExpressions.stream().map(indexAliases::get).filter(Objects::nonNull).toArray(AliasMetadata[]::new); } List aliases = null; for (AliasMetadata aliasMetadata : aliasCandidates) { if (requiredAlias.test(aliasMetadata)) { // If required - add it to the list of aliases if (aliases == null) { aliases = new ArrayList<>(); } aliases.add(aliasMetadata.alias()); } else { // If not, we have a non required alias for this index - no further checking needed return null; } } if (aliases == null) { return null; } return aliases.toArray(new String[0]); } /** * Resolves the search routing if in the expression aliases are used. If expressions point to concrete indices * or aliases with no routing defined the specified routing is used. * * @return routing values grouped by concrete index */ public Map> resolveSearchRouting(ClusterState state, @Nullable String routing, String... expressions) { List resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.emptyList(); Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, isSystemIndexAccessAllowed()); for (ExpressionResolver expressionResolver : expressionResolvers) { resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions); } // TODO: it appears that this can never be true? if (isAllIndices(resolvedExpressions)) { return resolveSearchRoutingAllIndices(state.metadata(), routing); } Map> routings = null; Set paramRouting = null; // List of indices that don't require any routing Set norouting = new HashSet<>(); if (routing != null) { paramRouting = Sets.newHashSet(Strings.splitStringByCommaToArray(routing)); } for (String expression : resolvedExpressions) { IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(expression); if (indexAbstraction != null && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) { IndexAbstraction.Alias alias = (IndexAbstraction.Alias) indexAbstraction; for (Tuple item : alias.getConcreteIndexAndAliasMetadatas()) { String concreteIndex = item.v1(); AliasMetadata aliasMetadata = item.v2(); if (!norouting.contains(concreteIndex)) { if (!aliasMetadata.searchRoutingValues().isEmpty()) { // Routing alias if (routings == null) { routings = new HashMap<>(); } Set r = routings.get(concreteIndex); if (r == null) { r = new HashSet<>(); routings.put(concreteIndex, r); } r.addAll(aliasMetadata.searchRoutingValues()); if (paramRouting != null) { r.retainAll(paramRouting); } if (r.isEmpty()) { routings.remove(concreteIndex); } } else { // Non-routing alias if (!norouting.contains(concreteIndex)) { norouting.add(concreteIndex); if (paramRouting != null) { Set r = new HashSet<>(paramRouting); if (routings == null) { routings = new HashMap<>(); } routings.put(concreteIndex, r); } else { if (routings != null) { routings.remove(concreteIndex); } } } } } } } else { // Index if (!norouting.contains(expression)) { norouting.add(expression); if (paramRouting != null) { Set r = new HashSet<>(paramRouting); if (routings == null) { routings = new HashMap<>(); } routings.put(expression, r); } else { if (routings != null) { routings.remove(expression); } } } } } if (routings == null || routings.isEmpty()) { return null; } return routings; } /** * Sets the same routing for all indices */ public Map> resolveSearchRoutingAllIndices(Metadata metadata, String routing) { if (routing != null) { Set r = Sets.newHashSet(Strings.splitStringByCommaToArray(routing)); Map> routings = new HashMap<>(); String[] concreteIndices = metadata.getConcreteAllIndices(); for (String index : concreteIndices) { routings.put(index, r); } return routings; } return null; } /** * Identifies whether the array containing index names given as argument refers to all indices * The empty or null array identifies all indices * * @param aliasesOrIndices the array containing index names * @return true if the provided array maps to all indices, false otherwise */ public static boolean isAllIndices(Collection aliasesOrIndices) { return aliasesOrIndices == null || aliasesOrIndices.isEmpty() || isExplicitAllPattern(aliasesOrIndices); } /** * Identifies whether the array containing index names given as argument explicitly refers to all indices * The empty or null array doesn't explicitly map to all indices * * @param aliasesOrIndices the array containing index names * @return true if the provided array explicitly maps to all indices, false otherwise */ static boolean isExplicitAllPattern(Collection aliasesOrIndices) { return aliasesOrIndices != null && aliasesOrIndices.size() == 1 && Metadata.ALL.equals(aliasesOrIndices.iterator().next()); } /** * Identifies whether the first argument (an array containing index names) is a pattern that matches all indices * * @param indicesOrAliases the array containing index names * @param concreteIndices array containing the concrete indices that the first argument refers to * @return true if the first argument is a pattern that maps to all available indices, false otherwise */ boolean isPatternMatchingAllIndices(Metadata metadata, String[] indicesOrAliases, String[] concreteIndices) { // if we end up matching on all indices, check, if its a wildcard parameter, or a "-something" structure if (concreteIndices.length == metadata.getConcreteAllIndices().length && indicesOrAliases.length > 0) { // we might have something like /-test1,+test1 that would identify all indices // or something like /-test1 with test1 index missing and IndicesOptions.lenient() if (indicesOrAliases[0].charAt(0) == '-') { return true; } // otherwise we check if there's any simple regex for (String indexOrAlias : indicesOrAliases) { if (Regex.isSimpleMatchPattern(indexOrAlias)) { return true; } } } return false; } /** * Determines whether or not system index access should be allowed in the current context. * * @return True if system index access should be allowed, false otherwise. */ public boolean isSystemIndexAccessAllowed() { return Booleans.parseBoolean(threadContext.getHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY), true); } /** * Context for the resolver. * * @opensearch.internal */ public static class Context { private final ClusterState state; private final IndicesOptions options; private final long startTime; private final boolean preserveAliases; private final boolean resolveToWriteIndex; private final boolean includeDataStreams; private final boolean preserveDataStreams; private final boolean isSystemIndexAccessAllowed; Context(ClusterState state, IndicesOptions options, boolean isSystemIndexAccessAllowed) { this(state, options, System.currentTimeMillis(), isSystemIndexAccessAllowed); } Context( ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex, boolean includeDataStreams, boolean isSystemIndexAccessAllowed ) { this( state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, false, isSystemIndexAccessAllowed ); } Context( ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex, boolean includeDataStreams, boolean preserveDataStreams, boolean isSystemIndexAccessAllowed ) { this( state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, preserveDataStreams, isSystemIndexAccessAllowed ); } Context(ClusterState state, IndicesOptions options, long startTime, boolean isSystemIndexAccessAllowed) { this(state, options, startTime, false, false, false, false, isSystemIndexAccessAllowed); } protected Context( ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex, boolean includeDataStreams, boolean preserveDataStreams, boolean isSystemIndexAccessAllowed ) { this.state = state; this.options = options; this.startTime = startTime; this.preserveAliases = preserveAliases; this.resolveToWriteIndex = resolveToWriteIndex; this.includeDataStreams = includeDataStreams; this.preserveDataStreams = preserveDataStreams; this.isSystemIndexAccessAllowed = isSystemIndexAccessAllowed; } public ClusterState getState() { return state; } public IndicesOptions getOptions() { return options; } public long getStartTime() { return startTime; } /** * This is used to prevent resolving aliases to concrete indices but this also means * that we might return aliases that point to a closed index. This is currently only used * by {@link #filteringAliases(ClusterState, String, Set)} since it's the only one that needs aliases */ boolean isPreserveAliases() { return preserveAliases; } /** * This is used to require that aliases resolve to their write-index. It is currently not used in conjunction * with preserveAliases. */ boolean isResolveToWriteIndex() { return resolveToWriteIndex; } public boolean includeDataStreams() { return includeDataStreams; } public boolean isPreserveDataStreams() { return preserveDataStreams; } /** * Used to determine if it is allowed to access system indices in this context (e.g. for this request). */ public boolean isSystemIndexAccessAllowed() { return isSystemIndexAccessAllowed; } } private interface ExpressionResolver { /** * Resolves the list of expressions into other expressions if possible (possible concrete indices and aliases, but * that isn't required). The provided implementations can also be left untouched. * * @return a new list with expressions based on the provided expressions */ List resolve(Context context, List expressions); } /** * Resolves alias/index name expressions with wildcards into the corresponding concrete indices/aliases * * @opensearch.internal */ static final class WildcardExpressionResolver implements ExpressionResolver { @Override public List resolve(Context context, List expressions) { IndicesOptions options = context.getOptions(); Metadata metadata = context.getState().metadata(); // only check open/closed since if we do not expand to open or closed it doesn't make sense to // expand to hidden if (options.expandWildcardsClosed() == false && options.expandWildcardsOpen() == false) { return expressions; } if (isEmptyOrTrivialWildcard(expressions)) { List resolvedExpressions = resolveEmptyOrTrivialWildcard(options, metadata); if (context.includeDataStreams()) { final IndexMetadata.State excludeState = excludeState(options); final Map dataStreamsAbstractions = metadata.getIndicesLookup() .entrySet() .stream() .filter(entry -> entry.getValue().getType() == IndexAbstraction.Type.DATA_STREAM) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); // dedup backing indices if expand hidden indices option is true Set resolvedIncludingDataStreams = new HashSet<>(resolvedExpressions); resolvedIncludingDataStreams.addAll( expand( context, excludeState, dataStreamsAbstractions, expressions.isEmpty() ? "_all" : expressions.get(0), options.expandWildcardsHidden() ) ); return new ArrayList<>(resolvedIncludingDataStreams); } return resolvedExpressions; } Set result = innerResolve(context, expressions, options, metadata); if (result == null) { return expressions; } if (result.isEmpty() && !options.allowNoIndices()) { IndexNotFoundException infe = new IndexNotFoundException((String) null); infe.setResources("index_or_alias", expressions.toArray(new String[0])); throw infe; } return new ArrayList<>(result); } private Set innerResolve(Context context, List expressions, IndicesOptions options, Metadata metadata) { Set result = null; boolean wildcardSeen = false; for (int i = 0; i < expressions.size(); i++) { String expression = expressions.get(i); if (Strings.isEmpty(expression)) { throw indexNotFoundException(expression); } validateAliasOrIndex(expression); if (aliasOrIndexExists(context, options, metadata, expression)) { if (result != null) { result.add(expression); } continue; } final boolean add; if (expression.charAt(0) == '-' && wildcardSeen) { add = false; expression = expression.substring(1); } else { add = true; } if (result == null) { // add all the previous ones... result = new HashSet<>(expressions.subList(0, i)); } if (Regex.isSimpleMatchPattern(expression) == false) { // TODO why does wildcard resolver throw exceptions regarding non wildcarded expressions? This should not be done here. if (options.ignoreUnavailable() == false) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(expression); if (indexAbstraction == null) { throw indexNotFoundException(expression); } else if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && options.ignoreAliases()) { throw aliasesNotSupportedException(expression); } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.includeDataStreams() == false) { throw indexNotFoundException(expression); } } if (add) { result.add(expression); } else { result.remove(expression); } continue; } final IndexMetadata.State excludeState = excludeState(options); final Map matches = matches(context, metadata, expression); Set expand = expand(context, excludeState, matches, expression, options.expandWildcardsHidden()); if (add) { result.addAll(expand); } else { result.removeAll(expand); } if (options.allowNoIndices() == false && matches.isEmpty()) { throw indexNotFoundException(expression); } if (Regex.isSimpleMatchPattern(expression)) { wildcardSeen = true; } } return result; } private static void validateAliasOrIndex(String expression) { // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API // does not exist and the path is interpreted as an expression. If the expression begins with an underscore, // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown // if the expression can't be found. if (expression.charAt(0) == '_') { throw new InvalidIndexNameException(expression, "must not start with '_'."); } } private static boolean aliasOrIndexExists(Context context, IndicesOptions options, Metadata metadata, String expression) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(expression); if (indexAbstraction == null) { return false; } // treat aliases as unavailable indices when ignoreAliases is set to true (e.g. delete index and update aliases api) if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && options.ignoreAliases()) { return false; } if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.includeDataStreams() == false) { return false; } return true; } private static IndexNotFoundException indexNotFoundException(String expression) { IndexNotFoundException infe = new IndexNotFoundException(expression); infe.setResources("index_or_alias", expression); return infe; } private static IndexMetadata.State excludeState(IndicesOptions options) { final IndexMetadata.State excludeState; if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) { excludeState = null; } else if (options.expandWildcardsOpen() && options.expandWildcardsClosed() == false) { excludeState = IndexMetadata.State.CLOSE; } else if (options.expandWildcardsClosed() && options.expandWildcardsOpen() == false) { excludeState = IndexMetadata.State.OPEN; } else { assert false : "this shouldn't get called if wildcards expand to none"; excludeState = null; } return excludeState; } public static Map matches(Context context, Metadata metadata, String expression) { if (Regex.isMatchAllPattern(expression)) { return filterIndicesLookup(context, metadata.getIndicesLookup(), null, context.getOptions()); } else if (expression.indexOf("*") == expression.length() - 1) { return suffixWildcard(context, metadata, expression); } else { return otherWildcard(context, metadata, expression); } } private static Map suffixWildcard(Context context, Metadata metadata, String expression) { assert expression.length() >= 2 : "expression [" + expression + "] should have at least a length of 2"; String fromPrefix = expression.substring(0, expression.length() - 1); char[] toPrefixCharArr = fromPrefix.toCharArray(); toPrefixCharArr[toPrefixCharArr.length - 1]++; String toPrefix = new String(toPrefixCharArr); SortedMap subMap = metadata.getIndicesLookup().subMap(fromPrefix, toPrefix); return filterIndicesLookup(context, subMap, null, context.getOptions()); } private static Map otherWildcard(Context context, Metadata metadata, String expression) { final String pattern = expression; return filterIndicesLookup( context, metadata.getIndicesLookup(), e -> Regex.simpleMatch(pattern, e.getKey()), context.getOptions() ); } private static Map filterIndicesLookup( Context context, SortedMap indicesLookup, Predicate> filter, IndicesOptions options ) { boolean shouldConsumeStream = false; Stream> stream = indicesLookup.entrySet().stream(); if (options.ignoreAliases()) { shouldConsumeStream = true; stream = stream.filter(e -> e.getValue().getType() != IndexAbstraction.Type.ALIAS); } if (filter != null) { shouldConsumeStream = true; stream = stream.filter(filter); } if (context.includeDataStreams() == false) { shouldConsumeStream = true; stream = stream.filter(e -> e.getValue().getType() != IndexAbstraction.Type.DATA_STREAM); } if (shouldConsumeStream) { return stream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } else { return indicesLookup; } } private static Set expand( Context context, IndexMetadata.State excludeState, Map matches, String expression, boolean includeHidden ) { Set expand = new HashSet<>(); for (Map.Entry entry : matches.entrySet()) { String aliasOrIndexName = entry.getKey(); IndexAbstraction indexAbstraction = entry.getValue(); if (indexAbstraction.isHidden() == false || includeHidden || implicitHiddenMatch(aliasOrIndexName, expression)) { if (context.isPreserveAliases() && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) { expand.add(aliasOrIndexName); } else { for (IndexMetadata meta : indexAbstraction.getIndices()) { if (excludeState == null || meta.getState() != excludeState) { expand.add(meta.getIndex().getName()); } } if (context.isPreserveDataStreams() && indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { expand.add(indexAbstraction.getName()); } } } } return expand; } private static boolean implicitHiddenMatch(String itemName, String expression) { return itemName.startsWith(".") && expression.startsWith(".") && Regex.isSimpleMatchPattern(expression); } private boolean isEmptyOrTrivialWildcard(List expressions) { return expressions.isEmpty() || (expressions.size() == 1 && (Metadata.ALL.equals(expressions.get(0)) || Regex.isMatchAllPattern(expressions.get(0)))); } private static List resolveEmptyOrTrivialWildcard(IndicesOptions options, Metadata metadata) { if (options.expandWildcardsOpen() && options.expandWildcardsClosed() && options.expandWildcardsHidden()) { return Arrays.asList(metadata.getConcreteAllIndices()); } else if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) { return Arrays.asList(metadata.getConcreteVisibleIndices()); } else if (options.expandWildcardsOpen() && options.expandWildcardsHidden()) { return Arrays.asList(metadata.getConcreteAllOpenIndices()); } else if (options.expandWildcardsOpen()) { return Arrays.asList(metadata.getConcreteVisibleOpenIndices()); } else if (options.expandWildcardsClosed() && options.expandWildcardsHidden()) { return Arrays.asList(metadata.getConcreteAllClosedIndices()); } else if (options.expandWildcardsClosed()) { return Arrays.asList(metadata.getConcreteVisibleClosedIndices()); } else { return Collections.emptyList(); } } } /** * A date math expression resolver. * * @opensearch.internal */ public static final class DateMathExpressionResolver implements ExpressionResolver { private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd"); private static final String EXPRESSION_LEFT_BOUND = "<"; private static final String EXPRESSION_RIGHT_BOUND = ">"; private static final char LEFT_BOUND = '{'; private static final char RIGHT_BOUND = '}'; private static final char ESCAPE_CHAR = '\\'; private static final char TIME_ZONE_BOUND = '|'; @Override public List resolve(final Context context, List expressions) { List result = new ArrayList<>(expressions.size()); for (String expression : expressions) { result.add(resolveExpression(expression, context)); } return result; } @SuppressWarnings("fallthrough") String resolveExpression(String expression, final Context context) { if (expression.startsWith(EXPRESSION_LEFT_BOUND) == false || expression.endsWith(EXPRESSION_RIGHT_BOUND) == false) { return expression; } boolean escape = false; boolean inDateFormat = false; boolean inPlaceHolder = false; final StringBuilder beforePlaceHolderSb = new StringBuilder(); StringBuilder inPlaceHolderSb = new StringBuilder(); final char[] text = expression.toCharArray(); final int from = 1; final int length = text.length - 1; for (int i = from; i < length; i++) { boolean escapedChar = escape; if (escape) { escape = false; } char c = text[i]; if (c == ESCAPE_CHAR) { if (escapedChar) { beforePlaceHolderSb.append(c); escape = false; } else { escape = true; } continue; } if (inPlaceHolder) { switch (c) { case LEFT_BOUND: if (inDateFormat && escapedChar) { inPlaceHolderSb.append(c); } else if (!inDateFormat) { inDateFormat = true; inPlaceHolderSb.append(c); } else { throw new OpenSearchParseException( "invalid dynamic name expression [{}]." + " invalid character in placeholder at position [{}]", new String(text, from, length), i ); } break; case RIGHT_BOUND: if (inDateFormat && escapedChar) { inPlaceHolderSb.append(c); } else if (inDateFormat) { inDateFormat = false; inPlaceHolderSb.append(c); } else { String inPlaceHolderString = inPlaceHolderSb.toString(); int dateTimeFormatLeftBoundIndex = inPlaceHolderString.indexOf(LEFT_BOUND); String mathExpression; String dateFormatterPattern; DateFormatter dateFormatter; final ZoneId timeZone; if (dateTimeFormatLeftBoundIndex < 0) { mathExpression = inPlaceHolderString; dateFormatter = DEFAULT_DATE_FORMATTER; timeZone = ZoneOffset.UTC; } else { if (inPlaceHolderString.lastIndexOf(RIGHT_BOUND) != inPlaceHolderString.length() - 1) { throw new OpenSearchParseException( "invalid dynamic name expression [{}]. missing closing `}`" + " for date math format", inPlaceHolderString ); } if (dateTimeFormatLeftBoundIndex == inPlaceHolderString.length() - 2) { throw new OpenSearchParseException( "invalid dynamic name expression [{}]. missing date format", inPlaceHolderString ); } mathExpression = inPlaceHolderString.substring(0, dateTimeFormatLeftBoundIndex); String patternAndTZid = inPlaceHolderString.substring( dateTimeFormatLeftBoundIndex + 1, inPlaceHolderString.length() - 1 ); int formatPatternTimeZoneSeparatorIndex = patternAndTZid.indexOf(TIME_ZONE_BOUND); if (formatPatternTimeZoneSeparatorIndex != -1) { dateFormatterPattern = patternAndTZid.substring(0, formatPatternTimeZoneSeparatorIndex); timeZone = DateUtils.of(patternAndTZid.substring(formatPatternTimeZoneSeparatorIndex + 1)); } else { dateFormatterPattern = patternAndTZid; timeZone = ZoneOffset.UTC; } dateFormatter = DateFormatter.forPattern(dateFormatterPattern); } DateFormatter formatter = dateFormatter.withZone(timeZone); DateMathParser dateMathParser = formatter.toDateMathParser(); Instant instant = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone); String time = formatter.format(instant); beforePlaceHolderSb.append(time); inPlaceHolderSb = new StringBuilder(); inPlaceHolder = false; } break; default: inPlaceHolderSb.append(c); } } else { switch (c) { case LEFT_BOUND: if (escapedChar) { beforePlaceHolderSb.append(c); } else { inPlaceHolder = true; } break; case RIGHT_BOUND: if (!escapedChar) { throw new OpenSearchParseException( "invalid dynamic name expression [{}]." + " invalid character at position [{}]. `{` and `}` are reserved characters and" + " should be escaped when used as part of the index name using `\\` (e.g. `\\{text\\}`)", new String(text, from, length), i ); } default: beforePlaceHolderSb.append(c); } } } if (inPlaceHolder) { throw new OpenSearchParseException( "invalid dynamic name expression [{}]. date math placeholder is open ended", new String(text, from, length) ); } if (beforePlaceHolderSb.length() == 0) { throw new OpenSearchParseException("nothing captured"); } return beforePlaceHolderSb.toString(); } } }