Browse Source

Do not create compound types if types differ by properties. (#6462)

* Do not create compound types if types differ by properties.

Before this change, compound types were created also for these types:

  * `text` -> `string{properties=[full-text-search]}`
  * `keyword` -> `string{properties=[enumerable]}`

This is incorrect, as these are the same (mapped) types, but vary only
on capabilities.

This change is now merging these two types into a single non-compound
type, with a property set that is the intersection of the given
properties of all merged types.

Fixes #6313.

* Adding license header.
pull/6474/head
Dennis Oelkers 1 month ago
parent
commit
477bf9355d

+ 9
- 2
graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/FieldTypesResource.java View File

@@ -41,6 +41,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -76,9 +77,14 @@ public class FieldTypesResource extends RestResource implements PluginRestResour
if (fieldTypes.size() == 1) {
return fieldTypes.iterator().next();
}
final String compoundFieldType = "compound(" + fieldTypes.stream()

final Set<String> distinctTypes = fieldTypes.stream()
.map(mappedFieldTypeDTO -> mappedFieldTypeDTO.type().type())
.collect(Collectors.joining(",")) + ")";
.sorted()
.collect(Collectors.toCollection(LinkedHashSet::new));
final String compoundFieldType = distinctTypes.size() > 1
? distinctTypes.stream().collect(Collectors.joining(",", "compound(", ")"))
: distinctTypes.stream().findFirst().orElse("unknown");
final ImmutableSet<String> commonProperties = fieldTypes.stream()
.map(mappedFieldTypeDTO -> mappedFieldTypeDTO.type().properties())
.reduce((s1, s2) -> Sets.intersection(s1, s2).immutableCopy())
@@ -86,6 +92,7 @@ public class FieldTypesResource extends RestResource implements PluginRestResour

final ImmutableSet<String> properties = ImmutableSet.<String>builder().addAll(commonProperties).add(PROP_COMPOUND_TYPE).build();
return MappedFieldTypeDTO.create(fieldName, createType(compoundFieldType, properties));

})
.collect(Collectors.toSet());


+ 130
- 0
graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/FieldTypesResourceTest.java View File

@@ -0,0 +1,130 @@
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see <http://www.gnu.org/licenses/>.
*/
package org.graylog.plugins.views.search.rest;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.shiro.subject.Subject;
import org.graylog2.indexer.fieldtypes.FieldTypeDTO;
import org.graylog2.indexer.fieldtypes.FieldTypeMapper;
import org.graylog2.indexer.fieldtypes.FieldTypes;
import org.graylog2.indexer.fieldtypes.IndexFieldTypesDTO;
import org.graylog2.indexer.fieldtypes.IndexFieldTypesService;
import org.graylog2.shared.security.RestPermissions;
import org.graylog2.streams.StreamService;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;

public class FieldTypesResourceTest {
@Rule
public final MockitoRule mockitoRule = MockitoJUnit.rule();

@Mock
private IndexFieldTypesService indexFieldTypesService;

@Mock
private StreamService streamService;

@Mock
private Subject currentSubject;

class FieldTypesTestResource extends FieldTypesResource {
FieldTypesTestResource(IndexFieldTypesService indexFieldTypesService, StreamService streamService, FieldTypeMapper fieldTypeMapper) {
super(indexFieldTypesService, streamService, fieldTypeMapper);
}

@Override
protected Subject getSubject() {
return currentSubject;
}
}

private FieldTypesResource fieldTypesResource;

@Before
public void setUp() throws Exception {
when(currentSubject.isPermitted(eq(RestPermissions.STREAMS_READ + ":*"))).thenReturn(true);
this.fieldTypesResource = new FieldTypesTestResource(indexFieldTypesService, streamService, new FieldTypeMapper());
}

private IndexFieldTypesDTO createIndexTypes(String indexId, String indexName, FieldTypeDTO... fieldTypes) {
return IndexFieldTypesDTO.create(indexId, indexName, Stream.of(fieldTypes).collect(Collectors.toSet()));
}

@Test
public void fieldsOfSameTypeDoNotReturnCompoundTypeIfPropertiesAreDifferent() {
final List<IndexFieldTypesDTO> fieldTypes = ImmutableList.of(
createIndexTypes(
"deadbeef",
"testIndex",
FieldTypeDTO.create("field1", "keyword"),
FieldTypeDTO.create("field2", "long")
),
createIndexTypes(
"affeaffe",
"testIndex2",
FieldTypeDTO.create("field1", "text"),
FieldTypeDTO.create("field2", "long")
)
);
when(indexFieldTypesService.findAll()).thenReturn(fieldTypes);

final Set<MappedFieldTypeDTO> result = this.fieldTypesResource.allFieldTypes();
assertThat(result).containsExactlyInAnyOrder(
MappedFieldTypeDTO.create("field2", FieldTypes.Type.createType("long", ImmutableSet.of("numeric", "enumerable"))),
MappedFieldTypeDTO.create("field1", FieldTypes.Type.createType("string", ImmutableSet.of("compound")))
);
}

@Test
public void fieldsOfDifferentTypesDoReturnCompoundType() {
final List<IndexFieldTypesDTO> fieldTypes = ImmutableList.of(
createIndexTypes(
"deadbeef",
"testIndex",
FieldTypeDTO.create("field1", "long"),
FieldTypeDTO.create("field2", "long")
),
createIndexTypes(
"affeaffe",
"testIndex2",
FieldTypeDTO.create("field1", "text"),
FieldTypeDTO.create("field2", "long")
)
);
when(indexFieldTypesService.findAll()).thenReturn(fieldTypes);

final Set<MappedFieldTypeDTO> result = this.fieldTypesResource.allFieldTypes();
assertThat(result).containsExactlyInAnyOrder(
MappedFieldTypeDTO.create("field2", FieldTypes.Type.createType("long", ImmutableSet.of("numeric", "enumerable"))),
MappedFieldTypeDTO.create("field1", FieldTypes.Type.createType("compound(long,string)", ImmutableSet.of("compound")))
);
}
}

Loading…
Cancel
Save