Browse Source

Upgrade to Elasticsearch 5.6 (#4249)

* Upgrade to Elasticsearch 5.6.4

Upgrade the internal ES dependency to Elasticsearch 5.6.4 and adapt our code to API changes.

* Remove Elasticsearch 2.x support

* Fix highlighting
tags/3.0.0-alpha.0
Jochen Schalanda 1 year ago
parent
commit
2d7b921dc2

+ 0
- 1
.travis.yml View File

@@ -51,5 +51,4 @@ env:
- secure: r0enVlGBPjqaGfNrD1b6YxgRaSp/v39ZMKQFs/C0RtRKliZ5LVo7kS4lL0tLQoqMAXJel0NgsVH6j1P6428uLQDxqKmCsNZiX0VS9K3Z0iq5NG4x/5QidxyXjHefSGmpquLmZbM094xhtvdou3NEfkJTKbxHu3ML9i9ScMn5+vA=
- secure: "OiqpeUFVXO/sWdSA4WZIrK5UvVUN4lG5YWm41e1P+6lvOAHpWeyX6fIs9Ndz12Vio/Wlt03ixzCAO/67UW4XGv/e3tClmvOSKt811Ml8wu3ALMxdV2218OkFXGzQ8AOnIRK1AAln3JdvBMHj8wFoUmGBaHJ7odqOxu3WC4m0Xr0="
matrix:
- GRAYLOG_ELASTICSEARCH_VERSION=2
- GRAYLOG_ELASTICSEARCH_VERSION=5

+ 1
- 7
graylog-project-parent/pom.xml View File

@@ -117,12 +117,6 @@
</exclusions>
</dependency>

<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>${jna.version}</version>
</dependency>

<dependency>
<groupId>org.kie</groupId>
<artifactId>kie-api</artifactId>
@@ -541,7 +535,7 @@
<dependency>
<groupId>com.github.joschi.nosqlunit</groupId>
<artifactId>nosqlunit-elasticsearch-http</artifactId>
<version>1.0.0-4+jackson</version>
<version>${nosqlunit-elasticsearch-http.version}</version>
<scope>test</scope>
</dependency>
<dependency>

+ 2
- 88
graylog2-server/pom.xml View File

@@ -65,7 +65,7 @@

<it.es.httpPort>19200</it.es.httpPort>
<it.es.transportPort>19300</it.es.transportPort>
<it.es.version>5.5.2</it.es.version>
<it.es.version>5.6.4</it.es.version>
<it.es.instanceCount>1</it.es.instanceCount>
<it.es.clusterName>test</it.es.clusterName>
<it.es.await>false</it.es.await>
@@ -274,11 +274,6 @@
<artifactId>jest</artifactId>
</dependency>

<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
</dependency>

<dependency>
<groupId>org.kie</groupId>
<artifactId>kie-api</artifactId>
@@ -356,10 +351,6 @@
<groupId>org.assertj</groupId>
<artifactId>assertj-joda-time</artifactId>
</dependency>
<dependency>
<groupId>com.lordofthejars</groupId>
<artifactId>nosqlunit-elasticsearch2</artifactId>
</dependency>
<dependency>
<groupId>com.lordofthejars</groupId>
<artifactId>nosqlunit-mongodb</artifactId>
@@ -772,83 +763,6 @@
</plugins>
</build>
</profile>
<profile>
<id>integration-tests-elasticsearch-2</id>
<activation>
<property>
<name>it.ElasticsearchVersion</name>
<value>2</value>
</property>
</activation>
<properties>
<it.es.version>2.4.6</it.es.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<skip>${it.es.skip}</skip>
</configuration>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
<version>2.3</version>
<configuration>
<skip>${it.es.skip}</skip>
<clusterName>${it.es.clusterName}</clusterName>
<httpPort>${it.es.httpPort}</httpPort>
<tcpPort>${it.es.transportPort}</tcpPort>
<autoCreateIndex>true</autoCreateIndex>
<configPath>${project.basedir}/src/test/resources/it/elasticsearch2</configPath>
<logsDirname>${project.build.directory}/elasticsearch2-logs</logsDirname>
<scriptFile/>
</configuration>
<executions>
<!-- The elasticsearch plugin is by default bound to the
pre-integration-test and post-integration-test phases -->
<execution>
<id>start-elasticsearch</id>
<phase>pre-integration-test</phase>
<goals>
<goal>start</goal>
</goals>
</execution>
<execution>
<id>stop-elasticsearch</id>
<phase>post-integration-test</phase>
<goals>
<goal>stop</goal>
</goals>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${it.es.version}</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>integration-tests-elasticsearch-5</id>
<activation>
@@ -859,7 +773,7 @@
</property>
</activation>
<properties>
<it.es.version>5.5.2</it.es.version>
<it.es.version>5.6.4</it.es.version>
</properties>
<build>
<plugins>

+ 0
- 56
graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping2.java View File

@@ -1,56 +0,0 @@
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see <http://www.gnu.org/licenses/>.
*/
package org.graylog2.indexer;

import com.google.common.collect.ImmutableMap;

import java.util.Map;

/**
* Representing the message type mapping in Elasticsearch 2.x. This is giving ES more
* information about what the fields look like and how it should analyze them.
*
* @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/2.4/mapping.html">Elasticsearch Reference / Mapping</a>
*/
public class IndexMapping2 extends IndexMapping {
@Override
protected Map<String, Map<String, Object>> fieldProperties(String analyzer) {
return ImmutableMap.of(
"message", analyzedString(analyzer),
"full_message", analyzedString(analyzer),
// http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html
// http://www.elasticsearch.org/guide/reference/mapping/date-format.html
"timestamp", typeTimeWithMillis(),
// to support wildcard searches in source we need to lowercase the content (wildcard search lowercases search term)
"source", analyzedString("analyzer_keyword"),
"streams", notAnalyzedString());
}

@Override
protected Map<String, Object> notAnalyzedString() {
return ImmutableMap.of(
"index", "not_analyzed",
"type", "string");
}

private Map<String, Object> analyzedString(String analyzer) {
return ImmutableMap.of(
"index", "analyzed",
"type", "string",
"analyzer", analyzer);
}
}

+ 1
- 3
graylog2-server/src/main/java/org/graylog2/indexer/IndexMappingFactory.java View File

@@ -33,9 +33,7 @@ public class IndexMappingFactory {

public IndexMapping createIndexMapping() {
final Version elasticsearchVersion = node.getVersion().orElseThrow(() -> new ElasticsearchException("Unable to retrieve Elasticsearch version."));
if (elasticsearchVersion.satisfies(">=2.1.0 & <5.0.0")) {
return new IndexMapping2();
} else if (elasticsearchVersion.satisfies("^5.0.0")) {
if (elasticsearchVersion.satisfies("^5.0.0")) {
return new IndexMapping5();
} else {
throw new ElasticsearchException("Unsupported Elasticsearch version: " + elasticsearchVersion);

+ 3
- 4
graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java View File

@@ -65,8 +65,8 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortParseElement;
import org.graylog2.audit.AuditActor;
import org.graylog2.audit.AuditEventSender;
import org.graylog2.indexer.ElasticsearchException;
@@ -141,7 +141,7 @@ public class Indices {
final String query = SearchSourceBuilder.searchSource()
.query(QueryBuilders.matchAllQuery())
.size(350)
.sort(SortBuilders.fieldSort(SortParseElement.DOC_FIELD_NAME))
.sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME))
.toString();

final Search request = new Search.Builder(query)
@@ -666,8 +666,7 @@ public class Indices {
* @see org.elasticsearch.search.aggregations.metrics.stats.Stats
*/
public IndexRangeStats indexRangeStatsOfIndex(String index) {
final FilterAggregationBuilder builder = AggregationBuilders.filter("agg")
.filter(QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP))
final FilterAggregationBuilder builder = AggregationBuilders.filter("agg", QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP))
.subAggregation(AggregationBuilders.min("ts_min").field(Message.FIELD_TIMESTAMP))
.subAggregation(AggregationBuilders.max("ts_max").field(Message.FIELD_TIMESTAMP))
.subAggregation(AggregationBuilders.terms("streams").field(Message.FIELD_STREAMS));

+ 1
- 1
graylog2-server/src/main/java/org/graylog2/indexer/results/FieldHistogramResult.java View File

@@ -107,7 +107,7 @@ public class FieldHistogramResult extends HistogramResult {
while (currentTime.getMillis() < maxTimestamp) {
final Map<String, Number> entry = results.get(currentTime.getMillis());

// advance timestamp by the interval's seconds value
// advance timestamp by the interval
currentTime.add(interval.getPeriod());

if (entry == null) {

+ 17
- 20
graylog2-server/src/main/java/org/graylog2/indexer/searches/Searches.java View File

@@ -43,13 +43,14 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.graylog2.Configuration;
import org.graylog2.database.NotFoundException;
import org.graylog2.indexer.ElasticsearchException;
@@ -296,8 +297,7 @@ public class Searches {
if (stackedFields.isEmpty()) {
// Wrap terms aggregation in a no-op filter to make sure the result structure is correct when not having
// stacked fields.
return AggregationBuilders.filter(AGG_FILTER)
.filter(QueryBuilders.matchAllQuery())
return AggregationBuilders.filter(AGG_FILTER, QueryBuilders.matchAllQuery())
.subAggregation(AggregationBuilders.terms(AGG_TERMS)
.field(field)
.size(size > 0 ? size : 50)
@@ -326,10 +326,9 @@ public class Searches {
filterQuery.must(QueryBuilders.existsQuery(f));
});

return AggregationBuilders.filter(AGG_FILTER)
.filter(filterQuery)
return AggregationBuilders.filter(AGG_FILTER, filterQuery)
.subAggregation(AggregationBuilders.terms(AGG_TERMS)
.script(new Script(scriptStringBuilder.toString(), ScriptService.ScriptType.INLINE, "painless", null))
.script(new Script(ScriptType.INLINE, "painless", scriptStringBuilder.toString(), Collections.emptyMap()))
.size(size > 0 ? size : 50)
.order(termsOrder));
}
@@ -393,9 +392,9 @@ public class Searches {
Sorting.Direction sorting) {
final Terms.Order termsOrder = sorting == Sorting.Direction.DESC ? Terms.Order.count(false) : Terms.Order.count(true);

final DateHistogramBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
final DateHistogramAggregationBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
.field(Message.FIELD_TIMESTAMP)
.interval(interval.toESInterval())
.dateHistogramInterval(interval.toESInterval())
.subAggregation(createTermsBuilder(field, stackedFields, size, termsOrder))
.subAggregation(AggregationBuilders.missing("missing").field(field));

@@ -484,16 +483,14 @@ public class Searches {
termsOrder = Terms.Order.count(true);
}

final FilterAggregationBuilder builder = AggregationBuilders.filter(AGG_FILTER)
final FilterAggregationBuilder builder = AggregationBuilders.filter(AGG_FILTER, standardAggregationFilters(range, filter))
.subAggregation(
AggregationBuilders.terms(AGG_TERMS_STATS)
.field(keyField)
.subAggregation(AggregationBuilders.stats(AGG_STATS).field(valueField))
.order(termsOrder)
.size(size)
)
.filter(standardAggregationFilters(range, filter));

);
searchSourceBuilder.aggregation(builder);

if (affectedIndices.isEmpty()) {
@@ -546,8 +543,7 @@ public class Searches {
searchSourceBuilder = filteredSearchRequest(query, filter, range);
}

final FilterAggregationBuilder filterBuilder = AggregationBuilders.filter(AGG_FILTER)
.filter(standardAggregationFilters(range, filter));
final FilterAggregationBuilder filterBuilder = AggregationBuilders.filter(AGG_FILTER, standardAggregationFilters(range, filter));
if (includeCount) {
searchSourceBuilder.aggregation(AggregationBuilders.count(AGG_VALUE_COUNT).field(field));
}
@@ -605,9 +601,9 @@ public class Searches {
}

public HistogramResult histogram(String query, DateHistogramInterval interval, String filter, TimeRange range) {
final DateHistogramBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
final DateHistogramAggregationBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
.field(Message.FIELD_TIMESTAMP)
.interval(interval.toESInterval());
.dateHistogramInterval(interval.toESInterval());

final SearchSourceBuilder searchSourceBuilder = filteredSearchRequest(query, filter, range)
.aggregation(histogramBuilder);
@@ -653,9 +649,9 @@ public class Searches {
TimeRange range,
boolean includeStats,
boolean includeCardinality) {
final DateHistogramBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
final DateHistogramAggregationBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
.field(Message.FIELD_TIMESTAMP)
.interval(interval.toESInterval());
.dateHistogramInterval(interval.toESInterval());

if (includeStats) {
dateHistogramBuilder.subAggregation(AggregationBuilders.stats(AGG_STATS).field(field));
@@ -797,11 +793,12 @@ public class Searches {
}

if (highlight && configuration.isAllowHighlighting()) {
searchSourceBuilder.highlighter()
final HighlightBuilder highlightBuilder = new HighlightBuilder()
.requireFieldMatch(false)
.field("*")
.fragmentSize(0)
.numOfFragments(0);
searchSourceBuilder.highlighter(highlightBuilder);
}

return searchSourceBuilder;

+ 1
- 4
graylog2-server/src/test/java/org/graylog2/ElasticsearchBase.java View File

@@ -39,7 +39,6 @@ import io.searchbox.indices.template.DeleteTemplate;
import io.searchbox.indices.template.GetTemplate;
import io.searchbox.indices.template.PutTemplate;
import org.graylog2.indexer.IndexMapping;
import org.graylog2.indexer.IndexMapping2;
import org.graylog2.indexer.IndexMapping5;
import org.junit.Rule;

@@ -120,12 +119,10 @@ public abstract class ElasticsearchBase {

protected IndexMapping indexMapping() {
switch (elasticsearchVersion.getMajorVersion()) {
case 2:
return new IndexMapping2();
case 5:
return new IndexMapping5();
default:
throw new IllegalStateException("Only Elasticsearch 2.x and 5.x are supported");
throw new IllegalStateException("Only Elasticsearch 5.x is supported");
}
}


+ 3
- 8
graylog2-server/src/test/java/org/graylog2/indexer/IndexMappingFactoryTest.java View File

@@ -18,8 +18,8 @@ package org.graylog2.indexer;

import com.github.zafarkhaja.semver.Version;
import org.graylog2.indexer.cluster.Node;
import org.junit.Rule;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -50,7 +50,7 @@ public class IndexMappingFactoryTest {
}

@Test
public void createIndexMappingFailsIfElasticsearchVersionIsTooLow() throws Exception {
public void createIndexMappingFailsIfElasticsearch1VersionIsTooLow() throws Exception {
when(node.getVersion()).thenReturn(Optional.of(Version.valueOf("1.7.3")));

assertThatThrownBy(indexMappingFactory::createIndexMapping)
@@ -70,7 +70,7 @@ public class IndexMappingFactoryTest {
}

@Test
public void createIndexMappingFailsIfElasticsearchVersionIsTooHigh() throws Exception {
public void createIndexMappingFailsIfElasticsearch6VersionIsTooHigh() throws Exception {
when(node.getVersion()).thenReturn(Optional.of(Version.valueOf("6.0.0")));

assertThatThrownBy(indexMappingFactory::createIndexMapping)
@@ -84,11 +84,6 @@ public class IndexMappingFactoryTest {
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][]{
{"2.1.0", IndexMapping2.class},
{"2.2.0", IndexMapping2.class},
{"2.3.0", IndexMapping2.class},
{"2.4.0", IndexMapping2.class},
{"2.4.5", IndexMapping2.class},
{"5.0.0", IndexMapping5.class},
{"5.1.0", IndexMapping5.class},
{"5.2.0", IndexMapping5.class},

+ 0
- 45
graylog2-server/src/test/java/org/graylog2/indexer/indices/IndicesIT.java View File

@@ -333,51 +333,6 @@ public class IndicesIT extends ElasticsearchBase {
assertThat(indices.indexCreationDate("index_missing")).isEmpty();
}

@Test
public void testIndexTemplateCanBeOverridden_Elasticsearch2() throws Exception {
assumeTrue(getElasticsearchVersion().getMajorVersion() == 2);

final String testIndexName = "graylog_override_template";
final String customTemplateName = "custom-template";

// Create custom index template
final Map<String, Object> customMapping = ImmutableMap.of(
"_source", ImmutableMap.of("enabled", false),
"properties", ImmutableMap.of("source",
ImmutableMap.of(
"type", "string",
"index", "not_analyzed"
)));
final Map<String, Object> templateSource = ImmutableMap.of(
"template", indexSet.getIndexWildcard(),
"order", 1,
"mappings", ImmutableMap.of(IndexMapping.TYPE_MESSAGE, customMapping)
);

putTemplate(customTemplateName, templateSource);

try {
// Validate existing index templates
final JsonNode existingTemplate = getTemplate(customTemplateName);
assertThat(existingTemplate.path(customTemplateName).isObject()).isTrue();

// Create index with custom template
indices.create(testIndexName, indexSet);
waitForGreenStatus(testIndexName);

// Check index mapping
final JsonNode indexMappings = getMapping(testIndexName);
final JsonNode mapping = indexMappings.path(testIndexName).path("mappings").path(IndexMapping.TYPE_MESSAGE);

assertThat(mapping.path("_source").path("enabled")).isEqualTo(BooleanNode.getFalse());
assertThat(mapping.path("properties").path("source").path("index")).isEqualTo(new TextNode("not_analyzed"));
} finally {
// Clean up
deleteTemplate(customTemplateName);
deleteIndex(testIndexName);
}
}

@Test
public void testIndexTemplateCanBeOverridden_Elasticsearch5() throws Exception {
assumeTrue(getElasticsearchVersion().getMajorVersion() == 5);

+ 4
- 5
graylog2-server/src/test/java/org/graylog2/indexer/searches/SearchesIT.java View File

@@ -684,11 +684,10 @@ public class SearchesIT extends ElasticsearchBase {
final DateTime to = from.plusHours(1);
final TimeRange timeRange = AbsoluteRange.create(from, to);
final RangeQueryBuilder queryBuilder = (RangeQueryBuilder) IndexHelper.getTimestampRangeFilter(timeRange);
assertThat(queryBuilder)
.isNotNull()
.hasFieldOrPropertyWithValue("name", "timestamp")
.hasFieldOrPropertyWithValue("from", Tools.buildElasticSearchTimeFormat(from))
.hasFieldOrPropertyWithValue("to", Tools.buildElasticSearchTimeFormat(to));
assertThat(queryBuilder).isNotNull();
assertThat(queryBuilder.fieldName()).isEqualTo("timestamp");
assertThat(queryBuilder.from()).isEqualTo(Tools.buildElasticSearchTimeFormat(from));
assertThat(queryBuilder.to()).isEqualTo(Tools.buildElasticSearchTimeFormat(to));
}

@Test

+ 2
- 2
pom.xml View File

@@ -93,7 +93,7 @@
<commons-io.version>2.6</commons-io.version>
<disruptor.version>3.3.7</disruptor.version>
<drools.version>6.5.0.Final</drools.version>
<elasticsearch.version>2.4.4</elasticsearch.version>
<elasticsearch.version>5.6.4</elasticsearch.version>
<jest.version>2.4.10+jackson</jest.version>
<gelfclient.version>1.4.1</gelfclient.version>
<grok.version>0.1.7-graylog</grok.version>
@@ -114,7 +114,6 @@
<jcip-annotations.version>1.0</jcip-annotations.version>
<jersey.version>2.25.1</jersey.version>
<jmte.version>4.0.0</jmte.version>
<jna.version>4.1.0</jna.version> <!-- for ES, make sure to use the version that ES uses -->
<joda-time.version>2.9.9</joda-time.version>
<json-path.version>2.4.0</json-path.version>
<kafka.version>0.9.0.1</kafka.version>
@@ -153,6 +152,7 @@
<junit.version>4.12</junit.version>
<mockito.version>2.12.0</mockito.version>
<nosqlunit.version>1.0.0-rc.5</nosqlunit.version>
<nosqlunit-elasticsearch-http.version>1.0.0-4+jackson</nosqlunit-elasticsearch-http.version>
<restassured.version>2.9.0</restassured.version>
<system-rules.version>1.16.1</system-rules.version>


Loading…
Cancel
Save