Commit d21ec4fc authored by Noel Alonso's avatar Noel Alonso
Browse files

Merge branch 'dev' into 'master' 

Actualiza versión

See merge request redmic-project/server/library/elasticsearch-lib!2
parents 7cb6f90c 231593c3

pom.xml

+43 −23
Original line number Diff line number Diff line 
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

<project xmlns="http://maven.apache.org/POM/4.0.0"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">



	<parent>

		<groupId>org.springframework.boot</groupId>

		<artifactId>spring-boot-starter-parent</artifactId>

    	<version>2.0.0.RELEASE</version>

		<version>2.1.0.RELEASE</version>

		<relativePath />

	</parent>
@@ -12,7 +13,7 @@ 
	<groupId>es.redmic.lib</groupId>

	<artifactId>elasticsearch-lib</artifactId>

	<packaging>jar</packaging>

	<version>0.6.0</version>

	<version>0.8.0</version>

	<name>elasticsearch-lib</name>



	<properties>
@@ -23,12 +24,14 @@ 
		<java.version>1.8</java.version>



		<!-- REDMIC -->

		<redmic.exceptions.version>0.6.0</redmic.exceptions.version>

		<redmic.models.version>0.6.0</redmic.models.version>

		<redmic.exceptions.version>0.7.0</redmic.exceptions.version>

		<redmic.models.version>0.8.0</redmic.models.version>



		<!-- Others -->

		<commons-io.version>1.3.2</commons-io.version>

		<elasticsearch.x-pack.version>5.6.1</elasticsearch.x-pack.version>

		<elasticsearch.version>6.5.1</elasticsearch.version>

		<jts-core.version>1.16.0</jts-core.version>

		<spatial4j.version>0.7</spatial4j.version>



		<!-- Environment variables -->

		<env.MAVEN_REPO_URL>https://artifactory.redmic.net/artifactory</env.MAVEN_REPO_URL>
@@ -52,12 +55,29 @@ 


		<dependency>

			<groupId>org.elasticsearch.client</groupId>

			<artifactId>x-pack-transport</artifactId>

			<version>${elasticsearch.x-pack.version}</version>

			<artifactId>elasticsearch-rest-high-level-client</artifactId>

		</dependency>



		<!-- Other -->



		<dependency>

			<groupId>org.locationtech.jts</groupId>

			<artifactId>jts-core</artifactId>

			<version>${jts-core.version}</version>

			<exclusions>

				<exclusion>

					<groupId>xerces</groupId>

					<artifactId>xercesImpl</artifactId>

				</exclusion>

			</exclusions>

		</dependency>



		<dependency>

			<groupId>org.locationtech.spatial4j</groupId>

			<artifactId>spatial4j</artifactId>

			<version>${spatial4j.version}</version>

		</dependency>



		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-io</artifactId>
@@ -66,8 +86,8 @@ 


		<!-- tests -->

		<dependency>

			<groupId>org.skyscreamer</groupId>

			<artifactId>jsonassert</artifactId>

			<groupId>org.springframework.boot</groupId>

			<artifactId>spring-boot-starter-test</artifactId>

			<scope>test</scope>

		</dependency>

	</dependencies>

src/main/java/es/redmic/elasticsearchlib/common/query/DataQueryUtils.java

+10 −19
Original line number Diff line number Diff line 
package es.redmic.elasticsearchlib.common.query;



import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;



import org.apache.lucene.search.join.ScoreMode;

import org.elasticsearch.common.geo.builders.ShapeBuilders;

import org.elasticsearch.common.geo.GeoPoint;

import org.elasticsearch.index.query.BoolQueryBuilder;

import org.elasticsearch.index.query.GeoShapeQueryBuilder;

import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder;

import org.elasticsearch.index.query.QueryBuilder;

import org.elasticsearch.index.query.QueryBuilders;

import org.elasticsearch.index.query.RangeQueryBuilder;

import org.elasticsearch.script.Script;

import org.elasticsearch.script.ScriptType;



import com.vividsolutions.jts.geom.Coordinate;



import es.redmic.exception.elasticsearch.ESBBoxQueryException;

import es.redmic.models.es.common.query.dto.BboxQueryDTO;

import es.redmic.models.es.common.query.dto.DataQueryDTO;

import es.redmic.models.es.common.query.dto.DateLimitsDTO;
@@ -62,19 +56,14 @@ public abstract class DataQueryUtils extends SimpleQueryUtils { 
		return getResultQuery(query);

	}



	public static GeoShapeQueryBuilder getBBoxQuery(BboxQueryDTO bbox) {

	public static GeoBoundingBoxQueryBuilder getBBoxQuery(BboxQueryDTO bbox) {



		if (bbox != null && bbox.getBottomRightLat() != null && bbox.getBottomRightLon() != null

				&& bbox.getTopLeftLat() != null && bbox.getTopLeftLon() != null) {



			Coordinate topLeft = new Coordinate(bbox.getTopLeftLon(), bbox.getTopLeftLat());

			Coordinate bottomRight = new Coordinate(bbox.getBottomRightLon(), bbox.getBottomRightLat());



			try {

				return QueryBuilders.geoShapeQuery("geometry", ShapeBuilders.newEnvelope(topLeft, bottomRight));

			} catch (IOException e) {

				throw new ESBBoxQueryException(e);

			}

			GeoPoint topLeft = new GeoPoint(bbox.getTopLeftLat(), bbox.getTopLeftLon());

			GeoPoint bottomRight = new GeoPoint(bbox.getBottomRightLat(), bbox.getBottomRightLon());

			return QueryBuilders.geoBoundingBoxQuery("geometry").setCorners(topLeft, bottomRight);

		}

		return null;

	}
@@ -104,7 +93,8 @@ public abstract class DataQueryUtils extends SimpleQueryUtils { 


		BoolQueryBuilder query = new BoolQueryBuilder();

		query.must(QueryBuilders.existsQuery((basePath != null) ? (basePath + "." + property) : property));

		query.must(QueryBuilders.scriptQuery(new Script(ScriptType.FILE, SCRIPT_ENGINE, scriptName, scriptParams)));

		// query.must(QueryBuilders.scriptQuery(new Script(ScriptType.FILE,

		// SCRIPT_ENGINE, scriptName, scriptParams)));



		return query;

	}
@@ -124,7 +114,8 @@ public abstract class DataQueryUtils extends SimpleQueryUtils { 
		BoolQueryBuilder query = new BoolQueryBuilder();

		query.must(QueryBuilders.nestedQuery(nestedPath,

				QueryBuilders.existsQuery(nestedPath + "." + basePath + "." + property), ScoreMode.Avg));

		query.must(QueryBuilders.scriptQuery(new Script(ScriptType.FILE, SCRIPT_ENGINE, scriptName, scriptParams)));

		// query.must(QueryBuilders.scriptQuery(new Script(ScriptType.FILE,

		// SCRIPT_ENGINE, scriptName, scriptParams)));



		return query;

	}

src/main/java/es/redmic/elasticsearchlib/common/repository/RBaseESRepository.java

+323 −113

File changed.

Preview size limit exceeded, changes collapsed.

src/main/java/es/redmic/elasticsearchlib/common/utils/ElasticPersistenceUtils.java

+59 −91
Original line number Diff line number Diff line 
package es.redmic.elasticsearchlib.common.utils;



import java.io.IOException;

import java.util.ArrayList;

import java.util.List;

import java.util.Map;

import java.util.concurrent.ExecutionException;



import org.apache.logging.log4j.LogManager;

import org.apache.logging.log4j.Logger;

import org.elasticsearch.action.bulk.BulkItemResponse;

import org.elasticsearch.action.bulk.BulkRequestBuilder;

import org.elasticsearch.action.bulk.BulkRequest;

import org.elasticsearch.action.bulk.BulkResponse;

import org.elasticsearch.action.delete.DeleteResponse;

import org.elasticsearch.action.delete.DeleteRequest;

import org.elasticsearch.action.index.IndexRequest;

import org.elasticsearch.action.index.IndexResponse;

import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;

import org.elasticsearch.action.update.UpdateRequest;

import org.elasticsearch.action.update.UpdateResponse;

import org.elasticsearch.client.RequestOptions;

import org.elasticsearch.common.xcontent.XContentBuilder;

import org.elasticsearch.rest.RestStatus;

import org.elasticsearch.script.Script;

import org.elasticsearch.script.ScriptType;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Component;
@@ -48,56 +46,59 @@ public class ElasticPersistenceUtils<TModel extends BaseES<?>> { 


		// @formatter:off

		

		IndexResponse result = ESProvider.getClient()

			.prepareIndex(index, type)

			.setSource(convertTModelToSource(model))

			.setId(id)

			.setRefreshPolicy(RefreshPolicy.IMMEDIATE)

				.execute()

					.actionGet();

		IndexRequest request = new IndexRequest(index, type)

				.source(convertTModelToSource(model))

				.id(id)

				.setRefreshPolicy(RefreshPolicy.IMMEDIATE);

		

		// @formatter:on



		if (!result.status().equals(RestStatus.CREATED)) {

		try {

			ESProvider.getClient().index(request, RequestOptions.DEFAULT);

			return new EventApplicationResult(true);

		} catch (IOException e) {

			logger.debug("Error indexando en " + index + " " + type);

			e.printStackTrace();

			return new EventApplicationResult(ExceptionType.ES_INDEX_DOCUMENT.toString());

		}



		return new EventApplicationResult(true);

	}



	public EventApplicationResult update(String index, String type, TModel model, String id) {



		UpdateRequest updateRequest = new UpdateRequest();

		updateRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);

		updateRequest.index(index);

		updateRequest.type(type);

		updateRequest.id(id);

		updateRequest.doc(convertTModelToSource(model));

		updateRequest.fetchSource(true);

		// @formatter:off



		UpdateRequest updateRequest = new UpdateRequest(index, type, id)

				.doc(convertTModelToSource(model))

				.fetchSource(false)

				.setRefreshPolicy(RefreshPolicy.IMMEDIATE);



		// @formatter:on



		try {

			ESProvider.getClient().update(updateRequest).get();

		} catch (InterruptedException | ExecutionException e) {

			ESProvider.getClient().update(updateRequest, RequestOptions.DEFAULT);



			return new EventApplicationResult(true);

		} catch (IOException e) {

			logger.debug("Error modificando el item con id " + id + " en " + index + " " + type);

			e.printStackTrace();

			return new EventApplicationResult(ExceptionType.ES_UPDATE_DOCUMENT.toString());

		}



		return new EventApplicationResult(true);

	}



	public EventApplicationResult update(String index, String type, String id, XContentBuilder doc) {



		UpdateRequest updateRequest = new UpdateRequest();

		updateRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);

		updateRequest.index(index);

		updateRequest.type(type);

		updateRequest.id(id);

		updateRequest.doc(doc);

		updateRequest.fetchSource(true);

		// @formatter:off

		

		UpdateRequest updateRequest = new UpdateRequest(index, type, id)

				.setRefreshPolicy(RefreshPolicy.IMMEDIATE)

				.doc(doc)

				.fetchSource(true);

		

		// @formatter:on



		try {

			ESProvider.getClient().update(updateRequest).get();

			ESProvider.getClient().update(updateRequest, RequestOptions.DEFAULT);

		} catch (Exception e) {

			logger.debug("Error modificando el item con id " + id + " en " + index + " " + type);

			return new EventApplicationResult(ExceptionType.ES_UPDATE_DOCUMENT.toString());
@@ -108,21 +109,15 @@ public class ElasticPersistenceUtils<TModel extends BaseES<?>> { 


	public EventApplicationResult delete(String index, String type, String id) {



		// @formatter:off



		DeleteResponse result = ESProvider.getClient()

			.prepareDelete(index, type, id)

			.setRefreshPolicy(RefreshPolicy.IMMEDIATE)

				.execute()

					.actionGet();



		// @formatter:on

		DeleteRequest deleteRequest = new DeleteRequest(index, type, id).setRefreshPolicy(RefreshPolicy.IMMEDIATE);



		if (!result.status().equals(RestStatus.OK)) {

		try {

			ESProvider.getClient().delete(deleteRequest, RequestOptions.DEFAULT);

			return new EventApplicationResult(true);

		} catch (IOException e) {

			logger.debug("Error borrando el item con id " + id + " en " + index + " " + type);

			return new EventApplicationResult(ExceptionType.DELETE_ITEM_EXCEPTION.toString());

		}

		return new EventApplicationResult(true);

	}



	@SuppressWarnings("unchecked")
@@ -166,55 +161,21 @@ public class ElasticPersistenceUtils<TModel extends BaseES<?>> { 
		return result;

	}



	public List<UpdateRequest> getUpdateScript(String[] index, String[] type, String id, Map<String, Object> fields,

			String scriptName) {



		return getUpdateScript(index, type, id, fields, scriptName, null, null);

	}



	public List<UpdateRequest> getUpdateScript(String[] index, String[] type, String id, Map<String, Object> fields,

			String scriptName, String parentId) {



		return getUpdateScript(index, type, id, fields, scriptName, parentId, null);

	}



	public List<UpdateRequest> getUpdateScript(String[] index, String[] type, String id, Map<String, Object> fields,

			String scriptName, String parentId, String grandParentId) {



		List<UpdateRequest> result = new ArrayList<UpdateRequest>();



		for (int i = 0; i < index.length; i++) {

			for (int j = 0; j < type.length; j++) {

				UpdateRequest updateRequest = new UpdateRequest();

				updateRequest.index(index[i]);

				updateRequest.type(type[j]);

				updateRequest.id(id);

				updateRequest.retryOnConflict(3);

				updateRequest.fetchSource(true);



				if (parentId != null)

					updateRequest.parent(parentId);



				if (grandParentId != null)

					updateRequest.routing(grandParentId);



				updateRequest.script(new Script(ScriptType.FILE, SCRIPT_ENGINE, scriptName, fields));

				updateRequest.retryOnConflict(2);

				result.add(updateRequest);

			}

		}

		return result;

	}



	public List<UpdateResponse> updateByBulk(List<UpdateRequest> listUpdates) {



		BulkRequestBuilder bulkRequest = ESProvider.getClient().prepareBulk();

		BulkRequest bulkRequest = new BulkRequest();



		for (int i = 0; i < listUpdates.size(); i++)

			bulkRequest.add(listUpdates.get(i));

		bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);



		BulkResponse bulkResponse = bulkRequest.execute().actionGet();

		BulkResponse bulkResponse;

		try {

			bulkResponse = ESProvider.getClient().bulk(bulkRequest, RequestOptions.DEFAULT);

		} catch (IOException e) {

			e.printStackTrace();

			throw new ESUpdateException("Error ejecutando modificación en batch");

		}



		if (bulkResponse.hasFailures()) {
@@ -238,12 +199,19 @@ public class ElasticPersistenceUtils<TModel extends BaseES<?>> { 


	public List<IndexResponse> indexByBulk(List<IndexRequest> listIndexs) {



		BulkRequestBuilder bulkRequest = ESProvider.getClient().prepareBulk();

		BulkRequest bulkRequest = new BulkRequest();



		for (int i = 0; i < listIndexs.size(); i++)

			bulkRequest.add(listIndexs.get(i));

		bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);

		BulkResponse bulkResponse = bulkRequest.execute().actionGet();



		BulkResponse bulkResponse;

		try {

			bulkResponse = ESProvider.getClient().bulk(bulkRequest, RequestOptions.DEFAULT);

		} catch (IOException e) {

			e.printStackTrace();

			throw new ESUpdateException("Error ejecutando indexación en batch");

		}



		if (bulkResponse.hasFailures())

			throw new ESUpdateException(bulkResponse.buildFailureMessage());

src/main/java/es/redmic/elasticsearchlib/common/utils/ElasticSearchUtils.java

+5 −5
Original line number Diff line number Diff line
@@ -15,11 +15,10 @@ import org.elasticsearch.action.get.GetResponse; 
import org.elasticsearch.action.get.MultiGetItemResponse;

import org.elasticsearch.action.get.MultiGetResponse;

import org.elasticsearch.action.search.SearchResponse;

import org.elasticsearch.common.Strings;

import org.elasticsearch.common.xcontent.ToXContentObject;

import org.elasticsearch.common.xcontent.XContentBuilder;

import org.elasticsearch.common.xcontent.XContentFactory;

import org.elasticsearch.common.xcontent.XContentHelper;

import org.elasticsearch.common.xcontent.XContentType;

import org.elasticsearch.index.query.BoolQueryBuilder;

import org.elasticsearch.index.query.QueryBuilder;

import org.elasticsearch.index.query.QueryBuilders;
@@ -27,7 +26,7 @@ import org.elasticsearch.search.SearchHit; 
import org.elasticsearch.search.SearchHits;

import org.elasticsearch.search.aggregations.AggregationBuilders;

import org.elasticsearch.search.aggregations.BaseAggregationBuilder;

import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;

import org.elasticsearch.search.aggregations.BucketOrder;

import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;

import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;

import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
@@ -139,6 +138,7 @@ public class ElasticSearchUtils { 
	/*

	 * Pasa la respuesta map

	 */

	@SuppressWarnings("unchecked")

	public static Map<String, Object> searchResponsetoObject(SearchResponse response) {



		XContentBuilder builder;
@@ -148,7 +148,7 @@ public class ElasticSearchUtils { 
			response.innerToXContent(builder, ToXContentObject.EMPTY_PARAMS);

			builder.endObject();



			return XContentHelper.convertToMap(builder.bytes(), true, XContentType.JSON).v2();

			return jMapper.readValue(Strings.toString(builder), Map.class);



		} catch (IOException e) {

			throw new ESParseException(e);
@@ -194,7 +194,7 @@ public class ElasticSearchUtils { 
					if (item.getMinCount() != 1)

						term.minDocCount(item.getMinCount());



					term.order(Order.term(true));

					term.order(BucketOrder.key(true));



					if (item.getSize() != null)

						term.size(item.getSize());