Skip to content

Commit

Permalink
Add basic support for field aliases in index mappings. (#31287)
Browse files Browse the repository at this point in the history
* Add basic support for field aliases through a new top-level mapper type.
* Add tests for queries, aggregations, sorting, and fetching doc values.
* Make sure we properly handle wildcard fields in query string queries.
* Allow for aliases when requesting suggestions.
* Allow for aliases when requesting highlights.
* Add a test for field capabilities.
  • Loading branch information
jtibshirani committed Jun 21, 2018
1 parent 99f503e commit d5642a0
Show file tree
Hide file tree
Showing 27 changed files with 1,282 additions and 60 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,8 @@ public DocumentMapper(MapperService mapperService, Mapping mapping) {
newFieldMappers.add(metadataMapper);
}
}
MapperUtils.collect(this.mapping.root, newObjectMappers, newFieldMappers);
MapperUtils.collect(this.mapping.root,
newObjectMappers, newFieldMappers, new ArrayList<>());

final IndexAnalyzers indexAnalyzers = mapperService.getIndexAnalyzers();
this.fieldMappers = new DocumentFieldMappers(newFieldMappers,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -459,13 +459,18 @@ private static ParseContext nestedContext(ParseContext context, ObjectMapper map
private static void parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
if (mapper instanceof ObjectMapper) {
parseObjectOrNested(context, (ObjectMapper) mapper);
} else {
FieldMapper fieldMapper = (FieldMapper)mapper;
} else if (mapper instanceof FieldMapper) {
FieldMapper fieldMapper = (FieldMapper) mapper;
Mapper update = fieldMapper.parse(context);
if (update != null) {
context.addDynamicMapper(update);
}
parseCopyFields(context, fieldMapper.copyTo().copyToFields());
} else if (mapper instanceof FieldAliasMapper) {
throw new IllegalArgumentException("Cannot write to a field alias [" + mapper.name() + "].");
} else {
throw new IllegalStateException("The provided mapper [" + mapper.name() + "] has an unrecognized type [" +
mapper.getClass().getSimpleName() + "].");
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.mapper;

import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;

import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;

/**
* A mapper for field aliases.
*
* A field alias has no concrete field mappings of its own, but instead points to another field by
* its path. Once defined, an alias can be used in place of the concrete field name in search requests.
*/
public final class FieldAliasMapper extends Mapper {
public static final String CONTENT_TYPE = "alias";

public static class Names {
public static final String PATH = "path";
}

private final String name;
private final String path;

public FieldAliasMapper(String simpleName,
String name,
String path) {
super(simpleName);
this.name = name;
this.path = path;
}

@Override
public String name() {
return name;
}

public String path() {
return path;
}

@Override
public Mapper merge(Mapper mergeWith) {
if (!(mergeWith instanceof FieldAliasMapper)) {
throw new IllegalArgumentException("Cannot merge a field alias mapping ["
+ name() + "] with a mapping that is not for a field alias.");
}
return mergeWith;
}

@Override
public Mapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
return this;
}

@Override
public Iterator<Mapper> iterator() {
return Collections.emptyIterator();
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject(simpleName())
.field("type", CONTENT_TYPE)
.field(Names.PATH, path)
.endObject();
}

public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
FieldAliasMapper.Builder builder = new FieldAliasMapper.Builder(name);
Object pathField = node.remove(Names.PATH);
String path = XContentMapValues.nodeStringValue(pathField, null);
if (path == null) {
throw new MapperParsingException("The [path] property must be specified for field [" + name + "].");
}
return builder.path(path);
}
}

public static class Builder extends Mapper.Builder<FieldAliasMapper.Builder, FieldAliasMapper> {
private String name;
private String path;

protected Builder(String name) {
super(name);
this.name = name;
}

public String name() {
return this.name;
}

public Builder path(String path) {
this.path = path;
return this;
}

public FieldAliasMapper build(BuilderContext context) {
String fullName = context.path().pathAsText(name);
return new FieldAliasMapper(name, fullName, path);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,30 +35,36 @@
*/
class FieldTypeLookup implements Iterable<MappedFieldType> {

/** Full field name to field type */
final CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType;
private final CopyOnWriteHashMap<String, String> aliasToConcreteName;

/** Create a new empty instance. */
FieldTypeLookup() {
fullNameToFieldType = new CopyOnWriteHashMap<>();
aliasToConcreteName = new CopyOnWriteHashMap<>();
}

private FieldTypeLookup(CopyOnWriteHashMap<String, MappedFieldType> fullName) {
this.fullNameToFieldType = fullName;
private FieldTypeLookup(CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType,
CopyOnWriteHashMap<String, String> aliasToConcreteName) {
this.fullNameToFieldType = fullNameToFieldType;
this.aliasToConcreteName = aliasToConcreteName;
}

/**
* Return a new instance that contains the union of this instance and the field types
* from the provided fields. If a field already exists, the field type will be updated
* to use the new mappers field type.
* from the provided mappers. If a field already exists, its field type will be updated
* to use the new type from the given field mapper. Similarly if an alias already
* exists, it will be updated to reference the field type from the new mapper.
*/
public FieldTypeLookup copyAndAddAll(String type, Collection<FieldMapper> fieldMappers) {
public FieldTypeLookup copyAndAddAll(String type,
Collection<FieldMapper> fieldMappers,
Collection<FieldAliasMapper> fieldAliasMappers) {
Objects.requireNonNull(type, "type must not be null");
if (MapperService.DEFAULT_MAPPING.equals(type)) {
throw new IllegalArgumentException("Default mappings should not be added to the lookup");
}

CopyOnWriteHashMap<String, MappedFieldType> fullName = this.fullNameToFieldType;
CopyOnWriteHashMap<String, String> aliases = this.aliasToConcreteName;

for (FieldMapper fieldMapper : fieldMappers) {
MappedFieldType fieldType = fieldMapper.fieldType();
Expand All @@ -75,7 +81,14 @@ public FieldTypeLookup copyAndAddAll(String type, Collection<FieldMapper> fieldM
}
}
}
return new FieldTypeLookup(fullName);

for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) {
String aliasName = fieldAliasMapper.name();
String fieldName = fieldAliasMapper.path();
aliases = aliases.copyAndPut(aliasName, fieldName);
}

return new FieldTypeLookup(fullName, aliases);
}

/**
Expand All @@ -92,7 +105,8 @@ private void checkCompatibility(MappedFieldType existingFieldType, MappedFieldTy

/** Returns the field for the given field */
public MappedFieldType get(String field) {
return fullNameToFieldType.get(field);
String resolvedField = aliasToConcreteName.getOrDefault(field, field);
return fullNameToFieldType.get(resolvedField);
}

/**
Expand All @@ -105,6 +119,11 @@ public Collection<String> simpleMatchToFullName(String pattern) {
fields.add(fieldType.name());
}
}
for (String aliasName : aliasToConcreteName.keySet()) {
if (Regex.simpleMatch(pattern, aliasName)) {
fields.add(aliasName);
}
}
return fields;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import com.carrotsearch.hppc.ObjectHashSet;
import com.carrotsearch.hppc.cursors.ObjectCursor;

import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
Expand Down Expand Up @@ -395,15 +394,16 @@ private synchronized Map<String, DocumentMapper> internalMerge(@Nullable Documen
// check basic sanity of the new mapping
List<ObjectMapper> objectMappers = new ArrayList<>();
List<FieldMapper> fieldMappers = new ArrayList<>();
List<FieldAliasMapper> fieldAliasMappers = new ArrayList<>();
Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers, fieldAliasMappers);
checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers, fullPathObjectMappers, fieldTypes);
checkObjectsCompatibility(objectMappers, fullPathObjectMappers);
checkPartitionedIndexConstraints(newMapper);

// update lookup data-structures
// this will in particular make sure that the merged fields are compatible with other types
fieldTypes = fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers);
fieldTypes = fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers, fieldAliasMappers);

for (ObjectMapper objectMapper : objectMappers) {
if (fullPathObjectMappers == this.fullPathObjectMappers) {
Expand Down Expand Up @@ -482,7 +482,7 @@ private boolean assertMappersShareSameFieldType() {
if (mapper != null) {
List<FieldMapper> fieldMappers = new ArrayList<>();
Collections.addAll(fieldMappers, mapper.mapping().metadataMappers);
MapperUtils.collect(mapper.root(), new ArrayList<>(), fieldMappers);
MapperUtils.collect(mapper.root(), new ArrayList<>(), fieldMappers, new ArrayList<>());
for (FieldMapper fieldMapper : fieldMappers) {
assert fieldMapper.fieldType() == fieldTypes.get(fieldMapper.name()) : fieldMapper.name();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,28 @@
enum MapperUtils {
;

/** Split mapper and its descendants into object and field mappers. */
public static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
/**
* Splits the provided mapper and its descendants into object, field, and field alias mappers.
*/
public static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
Collection<FieldMapper> fieldMappers,
Collection<FieldAliasMapper> fieldAliasMappers) {
if (mapper instanceof RootObjectMapper) {
// root mapper isn't really an object mapper
} else if (mapper instanceof ObjectMapper) {
objectMappers.add((ObjectMapper)mapper);
} else if (mapper instanceof FieldMapper) {
fieldMappers.add((FieldMapper)mapper);
} else if (mapper instanceof FieldAliasMapper) {
fieldAliasMappers.add((FieldAliasMapper) mapper);
} else {
throw new IllegalStateException("Unrecognized mapper type [" +
mapper.getClass().getSimpleName() + "].");
}


for (Mapper child : mapper) {
collect(child, objectMappers, fieldMappers);
collect(child, objectMappers, fieldMappers, fieldAliasMappers);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.IpFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MetadataFieldMapper;
import org.elasticsearch.index.mapper.NumberFieldMapper;
Expand Down Expand Up @@ -167,23 +168,27 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,
if (fieldSuffix != null && context.fieldMapper(fieldName + fieldSuffix) != null) {
fieldName = fieldName + fieldSuffix;
}
FieldMapper mapper = getFieldMapper(context.getMapperService(), fieldName);
if (mapper == null) {
// Unmapped fields are not ignored
fields.put(fieldOrPattern, weight);
continue;
}
if (acceptMetadataField == false && mapper instanceof MetadataFieldMapper) {
// Ignore metadata fields

MappedFieldType fieldType = context.getMapperService().fullName(fieldName);
if (fieldType == null) {
// Note that we don't ignore unmapped fields.
fields.put(fieldName, weight);
continue;
}

// Ignore fields that are not in the allowed mapper types. Some
// types do not support term queries, and thus we cannot generate
// a special query for them.
String mappingType = mapper.fieldType().typeName();
String mappingType = fieldType.typeName();
if (acceptAllTypes == false && ALLOWED_QUERY_MAPPER_TYPES.contains(mappingType) == false) {
continue;
}

// Ignore metadata fields.
FieldMapper mapper = getFieldMapper(context.getMapperService(), fieldName);
if (acceptMetadataField == false && mapper instanceof MetadataFieldMapper) {
continue;
}
fields.put(fieldName, weight);
}
checkForTooManyFields(fields);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.elasticsearch.index.mapper.BooleanFieldMapper;
import org.elasticsearch.index.mapper.CompletionFieldMapper;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.index.mapper.FieldAliasMapper;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.GeoPointFieldMapper;
import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
Expand Down Expand Up @@ -129,7 +130,9 @@ private Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mapperPlugi
mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser());
mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser());
mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser());
mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());

if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
if (highlightQuery == null) {
highlightQuery = context.parsedQuery().query();
}
HighlighterContext highlighterContext = new HighlighterContext(fieldName,
HighlighterContext highlighterContext = new HighlighterContext(fieldType.name(),
field, fieldType, context, hitContext, highlightQuery);

if ((highlighter.canHighlight(fieldType) == false) && fieldNameContainsWildcards) {
Expand All @@ -109,7 +109,11 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
}
HighlightField highlightField = highlighter.highlight(highlighterContext);
if (highlightField != null) {
highlightFields.put(highlightField.name(), highlightField);
// Note that we make sure to use the original field name in the response. This is because the
// original field could be an alias, and highlighter implementations may instead reference the
// concrete field it points to.
highlightFields.put(fieldName,
new HighlightField(fieldName, highlightField.fragments()));
}
}
}
Expand Down
Loading

0 comments on commit d5642a0

Please sign in to comment.