Skip to content

Commit

Permalink
fix(data): Optimisation of data search with a new searchDataListByTar…
Browse files Browse the repository at this point in the history
…gets service

OpenSILEX/opensilex-dev!1197
  • Loading branch information
Renaud Colin authored and Crejak committed Apr 5, 2024
1 parent 4d3d8f5 commit ed2a6c1
Show file tree
Hide file tree
Showing 47 changed files with 2,839 additions and 653 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.opensilex.OpenSilexModule;
import org.opensilex.core.config.SharedResourceInstanceItem;
import org.opensilex.core.data.dal.DataDAO;
import org.opensilex.core.data.dal.DataDaoV2;
import org.opensilex.core.device.dal.DeviceDAO;
import org.opensilex.core.event.dal.move.MoveEventDAO;
import org.opensilex.core.geospatial.dal.GeospatialDAO;
Expand All @@ -33,6 +34,7 @@
import org.opensilex.nosql.mongodb.MongoDBConfig;
import org.opensilex.nosql.mongodb.MongoDBService;
import org.opensilex.nosql.mongodb.MongoModel;
import org.opensilex.nosql.mongodb.service.v2.MongoDBServiceV2;
import org.opensilex.security.account.ModuleWithNosqlEntityLinkedToAccount;
import org.opensilex.server.exceptions.BadRequestException;
import org.opensilex.server.extensions.APIExtension;
Expand Down Expand Up @@ -220,6 +222,11 @@ public void install(boolean reset) throws Exception {

@Override
public void startup() throws Exception {
// Ensure index creation on application start (only in production)
if (!getOpenSilex().isTest() && !getOpenSilex().isReservedProfile()) {
MongoDBServiceV2 mongoDBServiceV2 = getOpenSilex().getServiceInstance(MongoDBServiceV2.DEFAULT_SERVICE, MongoDBServiceV2.class);
mongoDBServiceV2.createIndexes(DataDaoV2.COLLECTION_NAME, DataDaoV2.getIndexes());
}

}

Expand Down
393 changes: 262 additions & 131 deletions opensilex-core/src/main/java/org/opensilex/core/data/api/DataAPI.java

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import io.swagger.annotations.ApiModelProperty;
import org.opensilex.core.data.dal.DataDAO;
import org.opensilex.core.data.dal.DataModel;
import org.opensilex.security.user.api.UserGetDTO;
import org.opensilex.server.rest.validation.DateFormat;
import org.opensilex.server.rest.validation.ValidURI;
import org.opensilex.utils.ListWithPagination;
Expand All @@ -29,9 +28,6 @@
*/
public class DataGetDTO extends DataCreationDTO {

@JsonProperty("publisher")
private UserGetDTO publisher;

@JsonProperty("issued")
private OffsetDateTime publicationDate;

Expand All @@ -52,14 +48,6 @@ public String getTimezone() {
return timezone;
}

public UserGetDTO getPublisher() {
return publisher;
}

public void setPublisher(UserGetDTO publisher) {
this.publisher = publisher;
}

public OffsetDateTime getPublicationDate() {
return publicationDate;
}
Expand Down Expand Up @@ -98,7 +86,7 @@ public void setDate(Instant instant, String offset, Boolean isDateTime) {
* a {@link LocalDate} to be displayed correctly.
* </p>
* <p>
* The preferred way of performing this operation is by using {@link DataDAO#modelToDTO(DataModel)},
* The preferred way of performing this operation is by using {@link DataDAO#modelToGetDTO(DataModel)},
* or {@link DataDAO#modelListToDTO(ListWithPagination)} in the case of a list.
* </p>
*
Expand Down Expand Up @@ -140,7 +128,7 @@ public void fromModel(DataModel model, Set<URI> dateVariables) {
* a {@link LocalDate} to be displayed correctly.
* </p>
* <p>
* The preferred way of performing this operation is by using {@link DataDAO#modelToDTO(DataModel)},
* The preferred way of performing this operation is by using {@link DataDAO#modelToGetDTO(DataModel)},
* or {@link DataDAO#modelListToDTO(ListWithPagination)} in the case of a list.
* </p>
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
//******************************************************************************
// DataGetDTO.java
// OpenSILEX - Licence AGPL V3.0 - https://www.gnu.org/licenses/agpl-3.0.en.html
// Copyright © INRAE 2020
// Contact: anne.tireau@inrae.fr, pascal.neveu@inrae.fr
//******************************************************************************
package org.opensilex.core.data.api;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.opensilex.core.data.dal.DataModel;
import org.opensilex.security.user.api.UserGetDTO;

import java.net.URI;
import java.util.Set;

/**
*
* @author rcolin
* Specialized DTO which store detailed information about the publisher
*/
@JsonPropertyOrder({"uri", "date","timezone", "target", "variable", "value", "confidence", "provenance", "metadata", "publisher"})

public class DataGetDetailsDTO extends DataGetDTO {

@JsonProperty("publisher")
private UserGetDTO publisher;

public UserGetDTO getPublisher() {
return publisher;
}

public void setPublisher(UserGetDTO publisher) {
this.publisher = publisher;
}

public static DataGetDetailsDTO getDtoFromModel(DataModel model, Set<URI> dateVariables) {
DataGetDetailsDTO dto = new DataGetDetailsDTO();
dto.fromModel(model, dateVariables);
return dto;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* *************************************************************************************
* DataGetSearchDTO.java
* OpenSILEX - Licence AGPL V3.0 - https://www.gnu.org/licenses/agpl-3.0.en.html
* Copyright @ INRAE 2024
* Contact : user@inrae.fr, anne.tireau@inrae.fr, pascal.neveu@inrae.fr
* ************************************************************************************
*/

package org.opensilex.core.data.api;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.opensilex.core.data.dal.DataModel;

import java.net.URI;
import java.util.Set;

/**
*
* @author rcolin
* DTO which for data search
*/
@JsonPropertyOrder({"uri", "date","timezone", "target", "variable", "value", "confidence", "provenance", "metadata", "publisher"})
public class DataGetSearchDTO extends DataGetDTO {

@JsonProperty("publisher")
private URI publisher;

public URI getPublisher() {
return publisher;
}

public void setPublisher(URI publisher) {
this.publisher = publisher;
}

public static DataGetSearchDTO getDtoFromModel(DataModel model, Set<URI> dateVariables) {
DataGetSearchDTO dto = new DataGetSearchDTO();
dto.fromModel(model, dateVariables);
return dto;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,7 @@
import org.apache.jena.vocabulary.XSD;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.opensilex.core.data.api.DataComputedGetDTO;
import org.opensilex.core.data.api.CriteriaDTO;
import org.opensilex.core.data.api.DataExportDTO;
import org.opensilex.core.data.api.DataGetDTO;
import org.opensilex.core.data.api.SingleCriteriaDTO;
import org.opensilex.core.data.api.*;
import org.opensilex.core.data.dal.aggregations.DataTargetAggregateModel;
import org.opensilex.core.data.utils.MathematicalOperator;
import org.opensilex.core.data.utils.DataValidateUtils;
Expand All @@ -37,10 +33,9 @@
import org.opensilex.core.variable.dal.VariableModel;
import org.opensilex.fs.service.FileStorageService;
import org.opensilex.nosql.exceptions.NoSQLInvalidURIException;
import org.opensilex.nosql.mongodb.MongoDBConfig;
import org.opensilex.nosql.mongodb.MongoDBService;
import org.opensilex.security.account.dal.AccountDAO;
import org.opensilex.security.account.dal.AccountModel;
import org.opensilex.security.user.api.UserGetDTO;
import org.opensilex.server.response.ErrorResponse;
import org.opensilex.sparql.deserializer.SPARQLDeserializers;
import org.opensilex.sparql.deserializer.URIDeserializer;
Expand Down Expand Up @@ -166,7 +161,6 @@ public ListWithPagination<DataModel> search(
Integer pageSize) throws Exception {

Document filter = searchFilter(user, experiments, targets, variables, provenances, devices, startDate, endDate, confidenceMin, confidenceMax, metadata, operators);

return nosql.searchWithPagination(DataModel.class, DATA_COLLECTION_NAME, filter, orderByList, page, pageSize);
}

Expand Down Expand Up @@ -310,7 +304,7 @@ public List<DataModel> search(
return nosql.search(DataModel.class, DATA_COLLECTION_NAME, filter, orderByList);
}

public Document getSelectedAgents(List<URI> agents){
public Document getSelectedAgents(List< URI> agents){

//Get all data that have :
// provenance.provUsed.uri IN devices or operators URIs
Expand Down Expand Up @@ -1503,7 +1497,7 @@ public Set<URI> getDatafileProvenances(AccountModel user, List<URI> experiments,
* @return
* @throws Exception
*/
private Set<URI> getAllDateVariables() throws Exception {
public Set<URI> getAllDateVariables() throws Exception {
return new HashSet<>(sparql.searchURIs(VariableModel.class, null, selectBuilder -> {
Var uriVar = SPARQLQueryHelper.makeVar(VariableModel.URI_FIELD);
selectBuilder.addWhere(uriVar, Oeso.hasDataType.asNode(), XSD.date.asNode());
Expand All @@ -1519,35 +1513,29 @@ private Set<URI> getAllDateVariables() throws Exception {
* @return
* @throws Exception
*/
public DataGetDTO modelToDTO(DataModel model) throws Exception {
public DataGetDTO modelToGetDTO(DataModel model) throws Exception {
Set<URI> dateVariables = getAllDateVariables();
return DataGetDTO.getDtoFromModel(model, dateVariables);
}

/**
* Converts a list of {@link DataModel} to a lislt of {@link DataGetDTO}, with the correct conversion if the value is of
* Converts a list of {@link DataModel} to a list of {@link DataGetDTO}, with the correct conversion if the value is of
* type xsd:date. This method should be called as few times as possible (ideally only once), as it performs
* a SPARQL query.
*
* @param modelList
* @return
* @throws Exception
*/
public ListWithPagination<DataGetDTO> modelListToDTO(ListWithPagination<DataModel> modelList) throws Exception {
public ListWithPagination<DataGetSearchDTO> modelListToDTO(ListWithPagination<DataModel> modelList) throws Exception {
Set<URI> dateVariables = getAllDateVariables();
List<DataGetDTO> dtoList = new ArrayList<>();
modelList.getList().forEach(dataModel -> {
DataGetDTO dto = DataGetDTO.getDtoFromModel(dataModel, dateVariables);
if (Objects.nonNull(dataModel.getPublisher())) {
try {
dto.setPublisher(UserGetDTO.fromModel(new AccountDAO(sparql).get(dataModel.getPublisher())));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
dtoList.add(dto);
});
return new ListWithPagination<>(dtoList, modelList.getPage(), modelList.getPageSize(), modelList.getTotal());

List<DataGetSearchDTO> dtoList = modelList.getList()
.stream()
.map(model -> DataGetSearchDTO.getDtoFromModel(model, dateVariables))
.collect(Collectors.toList());

return new ListWithPagination<>(dtoList, modelList.getPage(), modelList.getPageSize(), modelList.getTotal(), modelList.getLimitCount());
}

/**
Expand Down
Loading

0 comments on commit ed2a6c1

Please sign in to comment.