[NAE-2494] Data projections

application-engine/src/main/java/com/netgrif/application/engine/actions/ActionApiImpl.java

@@ -124,11 +124,21 @@ public Page<Case> searchCases(String processIdentifier, Predicate predicate, Pag
         return searchCases(processIdentifier, predicate, pageable, new HashMap<>());
     }
 
+    @Override
+    public Page<Case> searchCases(String processIdentifier, Predicate predicate, Pageable pageable, Collection<String> dataFieldIds) {
+        return searchCases(processIdentifier, predicate, pageable, dataFieldIds, new HashMap<>());
+    }
+
     @Override
     public Page<Case> searchCases(List<String> elasticStringQueries, AuthPrincipalDto authPrincipalDto, Pageable pageable, Boolean isIntersection) {
         return searchCases(elasticStringQueries, authPrincipalDto, pageable, isIntersection, new HashMap<>());
     }
 
+    @Override
+    public Page<Case> searchCases(List<String> elasticStringQueries, AuthPrincipalDto authPrincipalDto, Pageable pageable, Boolean isIntersection, Collection<String> dataFieldIds) {
+        return searchCases(elasticStringQueries, authPrincipalDto, pageable, isIntersection, dataFieldIds, new HashMap<>());
+    }
+
     @Override
     public Long countCases(List<String> elasticStringQueries, AuthPrincipalDto authPrincipalDto, Boolean isIntersection) {
         return countCases(elasticStringQueries, authPrincipalDto, isIntersection, new HashMap<>());
@@ -140,6 +150,12 @@ public Page<Case> searchCases(String processIdentifier, Predicate predicate, Pag
         return workflowService.search(predicate, pageable);
     }
 
+    @Override
+    public Page<Case> searchCases(String processIdentifier, Predicate predicate, Pageable pageable, Collection<String> dataFieldIds, Map<String, String> params) {
+        log.debug("Searching cases for process identifier [{}] with predicate [{}], pageable [{}], dataFieldIds [{}]", processIdentifier, predicate, pageable, dataFieldIds);
+        return dataFieldIds == null ? workflowService.search(predicate, pageable) : workflowService.searchProjected(predicate, pageable, dataFieldIds);
+    }
+
     @Override
     public Page<Case> searchCases(List<String> elasticStringQueries, AuthPrincipalDto authPrincipalDto, Pageable pageable, Boolean isIntersection, Map<String, String> params) {
         log.debug("Searching cases for elastic queries [{}] with auth principal [{}], pageable [{}], intersect [{}]", elasticStringQueries, authPrincipalDto, pageable, isIntersection);
@@ -151,6 +167,17 @@ public Page<Case> searchCases(List<String> elasticStringQueries, AuthPrincipalDt
         return elasticCaseService.search(caseSearchRequests, loggedUser, pageable, locale, intersect);
     }
 
+    @Override
+    public Page<Case> searchCases(List<String> elasticStringQueries, AuthPrincipalDto authPrincipalDto, Pageable pageable, Boolean isIntersection, Collection<String> dataFieldIds, Map<String, String> params) {
+        log.debug("Searching cases for elastic queries [{}] with auth principal [{}], pageable [{}], intersect [{}], dataFieldIds [{}]", elasticStringQueries, authPrincipalDto, pageable, isIntersection, dataFieldIds);
+        boolean intersect = Boolean.TRUE.equals(isIntersection);
+        List<CaseSearchRequest> caseSearchRequests = elasticStringQueries.stream().map(query -> CaseSearchRequest.builder().query(query).build()).toList();
+        LoggedUser loggedUser = ActorTransformer.toLoggedUser(resolveAbstractUser(authPrincipalDto));
+        Locale locale = LocaleContextHolder.getLocale();
+        log.trace("Searching cases for elastic queries [{}] with auth principal [{}], pageable [{}], intersect [{}], locale [{}], dataFieldIds [{}]", elasticStringQueries, authPrincipalDto, pageable, isIntersection, locale, dataFieldIds);
+        return elasticCaseService.search(caseSearchRequests, loggedUser, pageable, locale, intersect, dataFieldIds);
+    }
+
     @Override
     public Long countCases(List<String> elasticStringQueries, AuthPrincipalDto authPrincipalDto, Boolean isIntersection, Map<String, String> params) {
         log.debug("Counting cases for elastic queries [{}] with auth principal [{}], intersect [{}]", elasticStringQueries, authPrincipalDto, isIntersection);
@@ -265,6 +292,57 @@ public Case findCase(String caseId) {
         return workflowService.findOne(caseId);
     }
 
+    @Override
+    public Case findCase(String caseId, Collection<String> dataFieldIds) {
+        log.debug("Finding case with id [{}] and data fields [{}]", caseId, dataFieldIds);
+        return dataFieldIds == null ? workflowService.findOne(caseId) : workflowService.findOneProjected(caseId, dataFieldIds);
+    }
+
+    @Override
+    public Case findCase(String caseId, Collection<String> dataFieldIds, boolean includeImmediateFields, boolean setPetriNet) {
+        log.debug("Finding case with id [{}] and data fields [{}], includeImmediateFields [{}], setPetriNet [{}]", caseId, dataFieldIds, includeImmediateFields, setPetriNet);
+        return dataFieldIds == null ? workflowService.findOne(caseId) : workflowService.findOneProjected(caseId, dataFieldIds, includeImmediateFields, setPetriNet);
+    }
+
+    @Override
+    public List<Case> findCases(List<String> caseIds) {
+        log.debug("Finding cases with ids [{}]", caseIds);
+        return workflowService.findAllById(caseIds);
+    }
+
+    @Override
+    public List<Case> findCases(List<String> caseIds, Collection<String> dataFieldIds) {
+        log.debug("Finding cases with ids [{}] and data fields [{}]", caseIds, dataFieldIds);
+        return dataFieldIds == null ? workflowService.findAllById(caseIds) : workflowService.findAllByIdProjected(caseIds, dataFieldIds);
+    }
+
+    @Override
+    public List<Case> findCases(List<String> caseIds, Collection<String> dataFieldIds, boolean includeImmediateFields, boolean setPetriNet) {
+        log.debug("Finding cases with ids [{}] and data fields [{}], includeImmediateFields [{}], setPetriNet [{}]", caseIds, dataFieldIds, includeImmediateFields, setPetriNet);
+        return dataFieldIds == null ? workflowService.findAllById(caseIds) : workflowService.findAllByIdProjected(caseIds, dataFieldIds, includeImmediateFields, setPetriNet);
+    }
+
+    @Override
+    public Case savePartialCase(Case partialCase) {
+        return savePartialCase(partialCase, new HashMap<>());
+    }
+
+    @Override
+    public Case savePartialCase(Case partialCase, Map<String, String> params) {
+        log.debug("Saving partial case with id [{}] and params [{}]", partialCase != null && partialCase.get_id() != null ? partialCase.getStringId() : null, params);
+        return workflowService.savePartial(partialCase);
+    }
+
+    @Override
+    public Case savePartial(Case partialCase) {
+        return savePartialCase(partialCase);
+    }
+
+    @Override
+    public Case savePartial(Case partialCase, Map<String, String> params) {
+        return savePartialCase(partialCase, params);
+    }
+
     @Override
     public Task findTask(String taskId) {
         log.debug("Finding task with id [{}]", taskId);

application-engine/src/main/java/com/netgrif/application/engine/configuration/ElasticServiceConfiguration.java

@@ -7,6 +7,7 @@
 import com.netgrif.application.engine.elastic.service.ElasticCaseService;
 import com.netgrif.application.engine.elastic.service.ElasticTaskService;
 import com.netgrif.application.engine.elastic.service.executors.Executor;
+import com.netgrif.application.engine.elastic.service.interfaces.IElasticCaseMappingService;
 import com.netgrif.application.engine.elastic.service.interfaces.IElasticCasePrioritySearch;
 import com.netgrif.application.engine.elastic.service.interfaces.IElasticCaseService;
 import com.netgrif.application.engine.elastic.service.interfaces.IElasticTaskService;
@@ -37,6 +38,7 @@ public class ElasticServiceConfiguration {
     private final IElasticCasePrioritySearch elasticCasePrioritySearch;
     private final ApplicationEventPublisher applicationEventPublisher;
     private final ElasticsearchClient elasticsearchClient;
+    private final IElasticCaseMappingService caseMappingService;
 
     @Bean
     @Primary
@@ -66,7 +68,8 @@ public IElasticCaseService elasticCaseService() {
                 workflowService,
                 elasticCasePrioritySearch,
                 applicationEventPublisher,
-                elasticsearchClient
+                elasticsearchClient,
+                caseMappingService
         );
     }
 
@@ -87,7 +90,8 @@ public IElasticCaseService reindexingTaskElasticCaseService() {
                 workflowService,
                 elasticCasePrioritySearch,
                 applicationEventPublisher,
-                elasticsearchClient
+                elasticsearchClient,
+                caseMappingService
         );
     }
 

application-engine/src/main/java/com/netgrif/application/engine/elastic/service/ElasticCaseMappingService.java

@@ -29,16 +29,23 @@ public ElasticCase transform(Case useCase) {
 
     protected void populateDataSet(ElasticCase transformedCase, Case useCase) {
         for (String id : useCase.getImmediateDataFields()) {
-            Optional<DataField> parsedValue = this.transformDataField(id, useCase);
-            parsedValue.ifPresent(dataField -> transformedCase.getDataSet().put(id, dataField));
+            if (useCase.isPartiallyLoaded()) {
+                if (useCase.getDataSet() != null && useCase.getDataSet().containsKey(id)) {
+                    Optional<DataField> parsedValue = this.transformDataField(id, useCase);
+                    transformedCase.getDataSet().put(id, parsedValue.orElse(null));
+                }
+            } else {
+                Optional<DataField> parsedValue = this.transformDataField(id, useCase);
+                parsedValue.ifPresent(dataField -> transformedCase.getDataSet().put(id, dataField));
+            }
         }
     }
 
     protected Optional<DataField> transformDataField(String fieldId, Case useCase) {
         Field<?> netField = useCase.getField(fieldId);
         com.netgrif.application.engine.objects.workflow.domain.DataField caseField = useCase.getDataField(fieldId);
 
-        if (caseField.getValue() == null) {
+        if (caseField == null || caseField.getValue() == null) {
             return Optional.empty();
         } else if (netField instanceof EnumerationMapField) {
             return this.transformEnumerationMapField(caseField, (EnumerationMapField) netField);

application-engine/src/main/java/com/netgrif/application/engine/elastic/service/ElasticCaseService.java

@@ -10,6 +10,7 @@
 import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
 import com.netgrif.application.engine.configuration.properties.DataConfigurationProperties;
 import com.netgrif.application.engine.elastic.domain.BulkOperationWrapper;
+import com.netgrif.application.engine.elastic.service.interfaces.IElasticCaseMappingService;
 import com.netgrif.application.engine.elastic.service.model.FullTextField;
 import com.netgrif.application.engine.objects.auth.domain.LoggedUser;
 import com.netgrif.application.engine.objects.elastic.domain.ElasticCase;
@@ -35,10 +36,13 @@
 import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate;
 import org.springframework.data.elasticsearch.client.elc.NativeQuery;
 import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
+import org.springframework.data.elasticsearch.core.SearchHit;
 import org.springframework.data.elasticsearch.core.SearchHitSupport;
 import org.springframework.data.elasticsearch.core.SearchHits;
+import org.springframework.data.elasticsearch.core.SearchPage;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
 import org.springframework.data.elasticsearch.core.query.*;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import java.util.*;
@@ -65,6 +69,7 @@ public class ElasticCaseService extends ElasticViewPermissionService implements
     protected ApplicationEventPublisher publisher;
     protected ElasticQueueManager caseElasticIndexQueueManager;
     protected ElasticQueueManager caseElasticDeleteQueueManager;
+    protected IElasticCaseMappingService caseMappingService;
 
     public ElasticCaseService(ElasticCaseRepository repository,
                               ElasticsearchTemplate template,
@@ -75,6 +80,19 @@ public ElasticCaseService(ElasticCaseRepository repository,
                               IElasticCasePrioritySearch elasticCasePrioritySearch,
                               ApplicationEventPublisher publisher,
                               ElasticsearchClient elasticsearchClient) {
+        this(repository, template, executors, elasticProperties, petriNetService, workflowService, elasticCasePrioritySearch, publisher, elasticsearchClient, null);
+    }
+
+    public ElasticCaseService(ElasticCaseRepository repository,
+                              ElasticsearchTemplate template,
+                              Executor executors,
+                              DataConfigurationProperties.ElasticsearchProperties elasticProperties,
+                              @Lazy IPetriNetService petriNetService,
+                              @Lazy IWorkflowService workflowService,
+                              IElasticCasePrioritySearch elasticCasePrioritySearch,
+                              ApplicationEventPublisher publisher,
+                              ElasticsearchClient elasticsearchClient,
+                              @Lazy IElasticCaseMappingService caseMappingService) {
         this.repository = repository;
         this.template = template;
         this.executors = executors;
@@ -85,7 +103,11 @@ public ElasticCaseService(ElasticCaseRepository repository,
         this.publisher = publisher;
         this.caseElasticIndexQueueManager = new ElasticQueueManager(elasticProperties, elasticsearchClient, publisher);
         this.caseElasticDeleteQueueManager = new ElasticQueueManager(elasticProperties, elasticsearchClient, publisher);
+        this.caseMappingService = caseMappingService;
+    }
 
+    public void setCaseMappingService(IElasticCaseMappingService caseMappingService) {
+        this.caseMappingService = caseMappingService;
     }
 
     @PreDestroy
@@ -128,8 +150,38 @@ public void indexNow(ElasticCase useCase) {
         index(useCase);
     }
 
+    @Async
+    @Override
+    public void update(ElasticCase useCase) {
+        if (!repository.existsById(useCase.getId())) {
+            log.debug("[{}]: Case \"{}\" not found in index for update, queueing full-case index", useCase.getId(), useCase.getTitle());
+            if (this.workflowService != null && this.caseMappingService != null) {
+                Case fullCase = this.workflowService.findOne(useCase.getId());
+                if (fullCase != null) {
+                    index(this.caseMappingService.transform(fullCase));
+                    return;
+                }
+            }
+        }
+        caseElasticIndexQueueManager.push(BulkOperationWrapper.builder()
+                .operation(createUpdateOperation(useCase))
+                .publishableEvent(new IndexCaseEvent(useCase))
+                .build());
+        log.debug("[{}]: Case \"{}\" queued for partial update", useCase.getId(), useCase.getTitle());
+    }
+
+    @Override
+    public void updateNow(ElasticCase useCase) {
+        update(useCase);
+    }
+
     @Override
     public Page<Case> search(List<CaseSearchRequest> requests, LoggedUser user, Pageable pageable, Locale locale, Boolean isIntersection) {
+        return search(requests, user, pageable, locale, isIntersection, null);
+    }
+
+    @Override
+    public Page<Case> search(List<CaseSearchRequest> requests, LoggedUser user, Pageable pageable, Locale locale, Boolean isIntersection, Collection<String> dataFieldIds) {
         if (requests == null) {
             throw new IllegalArgumentException("Request can not be null!");
         }
@@ -142,9 +194,10 @@ public Page<Case> search(List<CaseSearchRequest> requests, LoggedUser user, Page
         List<Case> casePage;
         long total;
         if (query != null) {
-            SearchHits<ElasticCase> hits = template.search(query, ElasticCase.class, IndexCoordinates.of(elasticProperties.getIndex().get(DataConfigurationProperties.ElasticsearchProperties.CASE_INDEX)));
-            Page<ElasticCase> indexedCases = (Page) SearchHitSupport.unwrapSearchHits(SearchHitSupport.searchPageFor(hits, query.getPageable()));
-            casePage = workflowService.findAllById(indexedCases.get().map(ElasticCase::getId).collect(Collectors.toList()));
+            SearchHits<com.netgrif.application.engine.adapter.spring.elastic.domain.ElasticCase> hits = template.search(query, com.netgrif.application.engine.adapter.spring.elastic.domain.ElasticCase.class, IndexCoordinates.of(elasticProperties.getIndex().get(DataConfigurationProperties.ElasticsearchProperties.CASE_INDEX)));
+            Page<com.netgrif.application.engine.adapter.spring.elastic.domain.ElasticCase> indexedCases = (Page) SearchHitSupport.unwrapSearchHits(SearchHitSupport.searchPageFor(hits, query.getPageable()));
+            List<String> ids = indexedCases.get().map(ElasticCase::getId).collect(Collectors.toList());
+            casePage = dataFieldIds == null ? workflowService.findAllById(ids) : workflowService.findAllByIdProjected(ids, dataFieldIds);
             total = indexedCases.getTotalElements();
             log.debug("Found [{}] total elements of page [{}]", casePage.size(), pageable.getPageNumber());
         } else {
@@ -191,7 +244,8 @@ protected NativeQuery buildQuery(List<CaseSearchRequest> requests, LoggedUser us
 
         NativeQueryBuilder builder = new NativeQueryBuilder()
                 .withQuery(query.build()._toQuery())
-                .withPageable(PageRequest.of(pageable.getPageNumber(), pageable.getPageSize()));
+                .withPageable(PageRequest.of(pageable.getPageNumber(), pageable.getPageSize()))
+                .withSourceFilter(new FetchSourceFilterBuilder().withIncludes("id").build());
 
         for (org.springframework.data.domain.Sort.Order o : pageable.getSort()) {
             builder.withSort(s -> s.field(f -> f
@@ -556,6 +610,15 @@ private BulkOperation createIndexOperation(ElasticCase useCase) {
                 .document(template.getElasticsearchConverter().mapObject(useCase))));
     }
 
+    private BulkOperation createUpdateOperation(ElasticCase useCase) {
+        return BulkOperation.of(op -> op.update(u -> u
+                .index(elasticProperties.getIndex().get(DataConfigurationProperties.ElasticsearchProperties.CASE_INDEX))
+                .id(useCase.getId())
+                .action(a -> a
+                        .doc(template.getElasticsearchConverter().mapObject(useCase))
+                )));
+    }
+
     private List<String> normalizeFullTextSearch(String fullText) {
         return Arrays.stream(fullText
                         .replace("\\", "")

application-engine/src/main/java/com/netgrif/application/engine/elastic/service/interfaces/IElasticCaseService.java

@@ -8,6 +8,7 @@
 import org.springframework.data.domain.Pageable;
 import org.springframework.scheduling.annotation.Async;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 
@@ -18,8 +19,20 @@ public interface IElasticCaseService {
 
     void indexNow(ElasticCase useCase);
 
+    @Async
+    void update(ElasticCase useCase);
+
+    void updateNow(ElasticCase useCase);
+
     Page<Case> search(List<CaseSearchRequest> requests, LoggedUser user, Pageable pageable, Locale locale, Boolean isIntersection);
 
+    /**
+     * Same as {@link #search(List, LoggedUser, Pageable, Locale, Boolean)}, but Mongo hydration loads only the given
+     * {@code dataSet} entries (plus automatically required ones). Returned cases are PARTIAL and READ-ONLY; persist
+     * changes with {@code IWorkflowService#savePartial}. A {@code null} {@code dataFieldIds} loads full cases.
+     */
+    Page<Case> search(List<CaseSearchRequest> requests, LoggedUser user, Pageable pageable, Locale locale, Boolean isIntersection, Collection<String> dataFieldIds);
+
     long count(List<CaseSearchRequest> requests, LoggedUser user, Locale locale, Boolean isIntersection);
 
     void remove(String caseId);