Commit 140e58fe authored by Noel Alonso's avatar Noel Alonso
Browse files

Merge branch 'feature-TimeSeriesMicroservice' into 'dev' 

Elimina control de  resultados erroneos

See merge request redmic-project/server/library/elasticsearch-lib!20
parents f3d13ee7 225a016f

pom.xml

+2 −2
Original line number Diff line number Diff line
@@ -13,13 +13,13 @@ 
	<groupId>es.redmic.lib</groupId>

	<artifactId>elasticsearch-lib</artifactId>

	<packaging>jar</packaging>

	<version>0.13.0</version>

	<version>0.13.0-feature-TimeSeriesMicroservice</version>

	<name>elasticsearch-lib</name>



	<properties>

		<!-- REDMIC -->

		<redmic.exceptions.version>0.10.0</redmic.exceptions.version>

		<redmic.models.version>0.11.0</redmic.models.version>

		<redmic.models.version>0.11.0-feature-cleanTimeSeries</redmic.models.version>



		<!-- Others -->

		<commons-io.version>1.3.2</commons-io.version>

src/main/java/es/redmic/elasticsearchlib/common/query/SeriesQueryUtils.java

0 → 100644
+170 −0
Original line number Diff line number Diff line 
package es.redmic.elasticsearchlib.common.query;



/*-

 * #%L

 * elasticsearch-lib

 * %%

 * Copyright (C) 2019 - 2021 REDMIC Project / Server

 * %%

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 * 

 *      http://www.apache.org/licenses/LICENSE-2.0

 * 

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 * #L%

 */



import java.util.ArrayList;

import java.util.HashSet;

import java.util.List;

import java.util.Set;



import org.elasticsearch.index.query.BoolQueryBuilder;

import org.elasticsearch.index.query.QueryBuilder;

import org.elasticsearch.index.query.QueryBuilders;

import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;



import es.redmic.exception.elasticsearch.ESHistogramIntervalQueryException;

import es.redmic.models.es.common.query.dto.DataQueryDTO;



public abstract class SeriesQueryUtils extends DataQueryUtils {

// @FORMATTER:OFF



	public static final BoolQueryBuilder INTERNAL_QUERY = null;



	public static final String DEFAULT_FIELD = "value";

	public static final String DATETIME_FIELD = "date";

	//public static final String PARENT = "geodata";

	//public static final String GRANDPARENT = "activity";



	// @FORMATTER:ON



	public static BoolQueryBuilder getQuery(DataQueryDTO queryDTO) {

		return getSeriesQuery(queryDTO, null, null);

	}



	public static BoolQueryBuilder getQuery(DataQueryDTO queryDTO, QueryBuilder internalQuery,

			QueryBuilder partialQuery) {

		return getSeriesQuery(queryDTO, internalQuery, partialQuery);

	}



	protected static BoolQueryBuilder getSeriesQuery(DataQueryDTO queryDTO, QueryBuilder internalQuery,

			QueryBuilder partialQuery) {



		BoolQueryBuilder query = getOrInitializeBaseQuery(getBaseQuery(queryDTO, internalQuery, partialQuery));



		addMustTermIfExist(query, getFlagQuery(queryDTO.getQFlags(), QFLAG_PROPERTY));

		addMustTermIfExist(query, getFlagQuery(queryDTO.getVFlags(), VFLAG_PROPERTY));

		addMustTermIfExist(query, getDateLimitsQuery(queryDTO.getDateLimits(), DATETIME_FIELD));

		addMustTermIfExist(query, getValueQuery(queryDTO.getValue(), VALUE_PROPERTY));

		addMustTermIfExist(query, getZQuery(Z_PROPERTY, SEARCH_BY_Z_RANGE_SCRIPT, queryDTO.getZ()));



		return getResultQuery(query);

	}



	public static BoolQueryBuilder getItemsQuery(String id) {



		List<String> ids = new ArrayList<>();

		ids.add(id);



		BoolQueryBuilder query = QueryBuilders.boolQuery();



		query.must(QueryBuilders.idsQuery().addIds(ids.toArray(new String[ids.size()])));



		return query;

	}



	/*-@SuppressWarnings("serial")

	public static BoolQueryBuilder getItemsQuery(String id, String parentId, String grandparentId,

			List<Long> accessibilityIds) {



		List<String> ids = new ArrayList<>();

		ids.add(id);

		return getItemsQuery(ids, parentId, grandparentId, accessibilityIds);

	}-*/



	/*-public static BoolQueryBuilder getItemsQuery(List<String> ids, String parentId, String grandparentId,

			List<Long> accessibilityIds) {



		BoolQueryBuilder query = QueryBuilders.boolQuery();



		if (accessibilityIds != null && accessibilityIds.size() > 0 && parentId != null && grandparentId != null)

			query.must(getQueryOnParentAndGrandparent(parentId, grandparentId, accessibilityIds));



		else if (accessibilityIds != null && accessibilityIds.size() > 0 && parentId == null && grandparentId == null)

			query.must(getAccessibilityQueryOnGrandparent(accessibilityIds));



		else if (accessibilityIds == null && parentId != null && grandparentId != null)

			query.must(getQueryByParentAndGrandparent(parentId, grandparentId));



		query.must(QueryBuilders.idsQuery().addIds(ids.toArray(new String[ids.size()])));



		return query;

	}-*/



	public static DateHistogramInterval getInterval(String interval) {



		if (!interval.matches("^\\d+[smhdwMqy]$"))

			throw new ESHistogramIntervalQueryException(interval);

		return new DateHistogramInterval(interval);

	}



	/*-public static QueryBuilder getHierarchicalQuery(DataQueryDTO queryDTO, String parentId, String grandparentId) {



		List<Long> accessibilityIds = queryDTO.getAccessibilityIds();



		if (parentId == null && grandparentId == null && (accessibilityIds == null || accessibilityIds.size() == 0))

			return null;



		if (parentId != null && grandparentId != null && (accessibilityIds == null || accessibilityIds.size() == 0))

			return getQueryByParentAndGrandparent(parentId, grandparentId);



		if (parentId == null || grandparentId == null && (accessibilityIds != null && accessibilityIds.size() > 0))

			return getAccessibilityQueryOnGrandparent(accessibilityIds);



		return getQueryOnParentAndGrandparent(parentId, grandparentId, accessibilityIds);

	}-*/



	/*-public static QueryBuilder getQueryByParentAndGrandparent(String parentId, String grandparentId) {

		return JoinQueryBuilders.hasParentQuery(PARENT,

				QueryBuilders.boolQuery()

						.must(JoinQueryBuilders.hasParentQuery(GRANDPARENT,

								QueryBuilders.boolQuery().must(QueryBuilders.termQuery("id", grandparentId)), true))

						.must(QueryBuilders.termQuery("_id", parentId)),

				true);

	}-*/



	/*-public static QueryBuilder getQueryOnParentAndGrandparent(String parentId, String grandparentId,

			List<Long> accessibilityIds) {



		return JoinQueryBuilders.hasParentQuery(PARENT,

				QueryBuilders.boolQuery()

						.must(JoinQueryBuilders.hasParentQuery(GRANDPARENT,

								QueryBuilders.boolQuery().must(QueryBuilders.termQuery("id", grandparentId))

										.must(getAccessibilityQuery(accessibilityIds)),

								true))

						.must(QueryBuilders.termQuery("_id", parentId)),

				true);

	}-*/



	/*-public static QueryBuilder getAccessibilityQueryOnGrandparent(List<Long> accessibilityIds) {



		return JoinQueryBuilders.hasParentQuery(PARENT,

				JoinQueryBuilders.hasParentQuery(GRANDPARENT, getAccessibilityQuery(accessibilityIds), true), true);

	}-*/



	public static Set<String> getFieldsExcludedOnQuery() {



		HashSet<String> fieldsExcludedOnQuery = new HashSet<>();



		fieldsExcludedOnQuery.add(ZRANGE_QUERY_FIELD);



		return fieldsExcludedOnQuery;

	}

}

src/main/java/es/redmic/elasticsearchlib/common/repository/RBaseESRepository.java

+55 −50
Original line number Diff line number Diff line
@@ -98,7 +98,7 @@ import es.redmic.models.es.common.query.dto.SuggestQueryDTO; 


public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO extends SimpleQueryDTO> {



	protected final static Logger LOGGER = LoggerFactory.getLogger(RBaseESRepository.class);

	protected static final Logger LOGGER = LoggerFactory.getLogger(RBaseESRepository.class);



	@Value("${redmic.elasticsearch.check.mappings}")

	private boolean checkMappings;
@@ -125,16 +125,16 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 


	protected Class<TModel> typeOfTModel;



	public RBaseESRepository() {

	protected RBaseESRepository() {

	}



	public RBaseESRepository(String[] index, String type, Boolean rollOverIndex) {

	protected RBaseESRepository(String[] index, String type, Boolean rollOverIndex) {

		this(index, type);

		ROLLOVER_INDEX = rollOverIndex;

	}



	@SuppressWarnings("unchecked")

	public RBaseESRepository(String[] index, String type) {

	protected RBaseESRepository(String[] index, String type) {

		this.INDEX = index;

		this.TYPE = type;
@@ -240,7 +240,7 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 
	 */

	private void prepareIndex(String index) {



		if (ROLLOVER_INDEX) {

		if (Boolean.TRUE.equals(ROLLOVER_INDEX)) {

			createTemplate(index);

		} else {

			createIndex(index);
@@ -313,7 +313,7 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 


	protected GetResponse getRequest(String id, String parentId) {



		LOGGER.debug("FindById en " + getIndex() + " " + getType() + " con id " + id);

		LOGGER.debug("FindById en {} {} con id {}", getIndex(), getType(), id);



		for (int i = 0; i < getIndex().length; i++) {
@@ -364,8 +364,8 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 
			partialQuery = QueryBuilders.boolQuery().must(termQuery);

		}



		LOGGER.debug("Suggest en " + getIndex() + " " + getType() + " con fields " + fields + " y texto " + text

				+ " y query interna " + partialQuery);

		LOGGER.debug("Suggest en {} {} con fields {} y texto {} y query interna {}",

			getIndex(), getType(), fields, text, partialQuery);



		SearchRequest searchRequest = new SearchRequest(INDEX);
@@ -405,14 +405,13 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 


	protected MultiGetResponse multigetRequest(MgetDTO dto, String parentId) {



		LOGGER.debug("Mget en " + getIndex() + " " + getType() + " con fields " + dto.getFields() + " e ids "

				+ dto.getIds());

		LOGGER.debug("Mget en {} {} con fields {} e ids {}", getIndex(), getType(), dto.getFields(), dto.getIds());



		MultiGetRequest request = new MultiGetRequest();



		FetchSourceContext fetchSourceContext;



		if (dto.getFields() == null || dto.getFields().size() == 0) {

		if (dto.getFields() == null || dto.getFields().isEmpty()) {

			fetchSourceContext = new FetchSourceContext(true);

		} else {

			String[] fieldsArray = dto.getFields().toArray(new String[dto.getFields().size()]);
@@ -458,8 +457,7 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 


	protected SearchResponse searchRequest(QueryBuilder query, SortBuilder<?> sort, List<String> returnFields) {



		LOGGER.debug("FindBy query en " + getIndex() + " " + getType() + " con query " + query.toString()

				+ " y ordenación " /* + sort.toString() */);

		LOGGER.debug("FindBy query en {} {} con query {} y ordenación.", getIndex(), getType(), query.toString() /* + sort.toString() */);



		Integer size = getCount(QueryBuilders.boolQuery().filter(query));
@@ -468,7 +466,7 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

		searchSourceBuilder.query(query).size(size).sort(sort);



		if (returnFields != null && returnFields.size() > 0) {

		if (returnFields != null && !returnFields.isEmpty()) {

			searchSourceBuilder.fetchSource(ElasticSearchUtils.getReturnFields(returnFields), null);

		}
@@ -493,8 +491,6 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 


	protected SearchResponse searchRequest(TQueryDTO queryDTO, QueryBuilder serviceQuery) {



		LOGGER.debug("Find en " + getIndex() + " " + getType() + " con queryDTO " + queryDTO + " y query interna ");



		SearchRequest searchRequest = new SearchRequest(getIndex());



		SearchSourceBuilder requestBuilder = searchRequestBuilder(queryDTO, serviceQuery);
@@ -520,25 +516,12 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 


	protected SearchSourceBuilder searchRequestBuilder(TQueryDTO queryDTO, QueryBuilder serviceQuery) {



		LOGGER.debug("Find en " + getIndex() + " " + getType() + " con queryDTO " + queryDTO + " y query interna ");



		QueryBuilder termQuery = getTermQuery(queryDTO.getTerms());



		BoolQueryBuilder partialQuery = null;

		if (serviceQuery != null) {

			partialQuery = QueryBuilders.boolQuery().must(serviceQuery);

		}



		if (termQuery != null && partialQuery != null) {

			partialQuery.must(termQuery);

		} else if (termQuery != null) {

			partialQuery = QueryBuilders.boolQuery().must(termQuery);

		}



		BoolQueryBuilder queryBuilder = getQuery(queryDTO, getInternalQuery(), partialQuery);

		LOGGER.debug("Find en {} {} con queryDTO {} y query interna.", getIndex(), getType(), queryDTO);



		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();



		BoolQueryBuilder queryBuilder = getQueryBuilder(queryDTO, serviceQuery);



		searchSourceBuilder.query(queryBuilder);



		QueryBuilder postFilter = getPostFilter(queryDTO.getPostFilter());
@@ -547,7 +530,7 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 
			searchSourceBuilder.postFilter(postFilter);

		}



		List<BaseAggregationBuilder> aggs = getAggs(queryDTO.getAggs());

		List<BaseAggregationBuilder> aggs = getAggs(queryDTO);



		if (aggs != null) {

			for (BaseAggregationBuilder term : aggs) {
@@ -569,19 +552,37 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 


		List<SortBuilder<?>> sorts = getSorts(queryDTO.getSorts());

		// Finalmente solo en caso de tener una ordenación se añade a la request

		if (sorts != null && sorts.size() > 0) {

		if (sorts != null && !sorts.isEmpty()) {

			for (int i = 0; i < sorts.size(); i++)

				searchSourceBuilder.sort(sorts.get(i));

		}



		List<String> returnFields = queryDTO.getReturnFields();

		if (returnFields != null && returnFields.size() > 0) {

		if (returnFields != null && !returnFields.isEmpty()) {

			searchSourceBuilder.fetchSource(ElasticSearchUtils.getReturnFields(returnFields), null);

		}



		return searchSourceBuilder;

	}



	protected BoolQueryBuilder getQueryBuilder(TQueryDTO queryDTO, QueryBuilder serviceQuery) {



		QueryBuilder termQuery = getTermQuery(queryDTO.getTerms());



		BoolQueryBuilder partialQuery = null;

		if (serviceQuery != null) {

			partialQuery = QueryBuilders.boolQuery().must(serviceQuery);

		}



		if (termQuery != null && partialQuery != null) {

			partialQuery.must(termQuery);

		} else if (termQuery != null) {

			partialQuery = QueryBuilders.boolQuery().must(termQuery);

		}



		return getQuery(queryDTO, getInternalQuery(), partialQuery);

	}



	private List<SortBuilder<?>> getSorts(List<SortDTO> sortDTOList) {



		// Se obtiene la ordenación enviada desde el exterior
@@ -698,6 +699,10 @@ public abstract class RBaseESRepository<TModel extends BaseES<?>, TQueryDTO exte 
	 * @return aggs de elastic.

	 */



	protected List<BaseAggregationBuilder> getAggs(TQueryDTO queryDTO) {

		return getAggs(queryDTO.getAggs());

	}



	protected List<BaseAggregationBuilder> getAggs(List<AggsPropertiesDTO> aggs) {

		return ElasticSearchUtils.getAggs(aggs);

	}

src/main/java/es/redmic/elasticsearchlib/common/utils/ElasticSearchUtils.java

+10 −8
Original line number Diff line number Diff line
@@ -168,8 +168,10 @@ public class ElasticSearchUtils { 
			response.innerToXContent(builder, ToXContentObject.EMPTY_PARAMS);

			builder.endObject();



			return jMapper.readValue(Strings.toString(builder), Map.class);



			Map<String, Object> result = jMapper.readValue(Strings.toString(builder), Map.class);

			// Elimina los fallos por tratarse de gran cantidad de datos que no interesan en el cliente.

			((Map<String, Object>) result.get("_shards")).remove("failures");

			return result;

		} catch (IOException e) {

			throw new ESParseException(e);

		}

src/main/java/es/redmic/elasticsearchlib/data/repository/RDataESRepository.java

+6 −6
Original line number Diff line number Diff line
@@ -46,14 +46,14 @@ import es.redmic.models.es.data.common.model.DataSearchWrapper; 
public abstract class RDataESRepository<TModel extends BaseES<?>, TQueryDTO extends SimpleQueryDTO>

		extends RBaseESRepository<TModel, TQueryDTO> {



	public RDataESRepository() {

	protected RDataESRepository() {

	}



	public RDataESRepository(String[] index, String type) {

	protected RDataESRepository(String[] index, String type) {

		super(index, type);

	}



	public RDataESRepository(String[] index, String type, Boolean rollOverIndex) {

	protected RDataESRepository(String[] index, String type, Boolean rollOverIndex) {

		super(index, type, rollOverIndex);

	}