1623 - added additional filterState last_runs_statuses, created API for generating SearchFacetsData

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataQualityRunsController.java

@@ -3,7 +3,11 @@
 import java.util.List;
 import lombok.RequiredArgsConstructor;
 import org.opendatadiscovery.oddplatform.api.contract.api.DataQualityRunsApi;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityBaseSearchForm;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityLatestRunSearchForm;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityResults;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityTableHealthSearchForm;
+import org.opendatadiscovery.oddplatform.api.contract.model.SearchFacetsData;
 import org.opendatadiscovery.oddplatform.service.DataQualityRunsService;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RestController;
@@ -31,4 +35,27 @@ public Mono<ResponseEntity<DataQualityResults>> getDataQualityTestsRuns(final Li
                 deNamespaceIds, deDatasourceIds, deOwnerIds, deTitleIds, deTagIds)
             .map(ResponseEntity::ok);
     }
+
+    @Override
+    public Mono<ResponseEntity<SearchFacetsData>> createDataQualityLatestRunsSearch(
+        final Mono<DataQualityLatestRunSearchForm> dataQualityLatestRunSearchForm,
+        final ServerWebExchange exchange) {
+        return dataQualityLatestRunSearchForm.flatMap(service::createDataQualityLatestRunsSearch)
+            .map(ResponseEntity::ok);
+    }
+
+    @Override
+    public Mono<ResponseEntity<SearchFacetsData>> createDataQualityTableHealthSearch(
+        final Mono<DataQualityTableHealthSearchForm> dataQualityTableHealthSearchForm,
+        final ServerWebExchange exchange) {
+        return dataQualityTableHealthSearchForm.flatMap(service::createDataQualityTableHealthSearch)
+            .map(ResponseEntity::ok);
+    }
+
+    @Override
+    public Mono<ResponseEntity<SearchFacetsData>> createDataQualityMonitoredTablesSearch(
+        final Mono<DataQualityBaseSearchForm> dataQualityBaseSearchForm, final ServerWebExchange exchange) {
+        return dataQualityBaseSearchForm.flatMap(service::createDataQualityMonitoredTablesSearch)
+            .map(ResponseEntity::ok);
+    }
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityQualityStatusDto.java

@@ -0,0 +1,50 @@
+package org.opendatadiscovery.oddplatform.dto;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import lombok.Getter;
+import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRunStatus;
+
+import static java.util.function.Function.identity;
+
+@Getter
+public enum DataEntityQualityStatusDto {
+    HEALTHY(1, "HEALTHY",
+        List.of(DataEntityRunStatus.SUCCESS),
+        List.of(DataEntityRunStatus.SKIPPED, DataEntityRunStatus.ABORTED, DataEntityRunStatus.UNKNOWN,
+            DataEntityRunStatus.FAILED, DataEntityRunStatus.BROKEN)),
+    ERROR(2, "ERROR",
+        List.of(DataEntityRunStatus.FAILED, DataEntityRunStatus.BROKEN),
+        List.of()),
+    WARNING(3, "WARNING",
+        List.of(DataEntityRunStatus.SKIPPED, DataEntityRunStatus.ABORTED, DataEntityRunStatus.UNKNOWN),
+        List.of(DataEntityRunStatus.FAILED, DataEntityRunStatus.BROKEN)),
+    ALL(4, "ALL", List.of(DataEntityRunStatus.values()), List.of()),
+    NOT_MONITORED(5, "NOT_MONITORED", List.of(), List.of(DataEntityRunStatus.values()));
+
+    private final short id;
+    private final String status;
+    private final List<DataEntityRunStatus> allowedRunStatuses;
+    private final List<DataEntityRunStatus> notAllowedRunStatuses;
+
+    DataEntityQualityStatusDto(final int id,
+                               final String status,
+                               final List<DataEntityRunStatus> allowedRunStatuses,
+                               final List<DataEntityRunStatus> notAllowedRunStatuses) {
+        this.id = (short) id;
+        this.status = status;
+        this.allowedRunStatuses = allowedRunStatuses;
+        this.notAllowedRunStatuses = notAllowedRunStatuses;
+    }
+
+    private static final Map<String, DataEntityQualityStatusDto> MAP = Arrays
+        .stream(DataEntityQualityStatusDto.values())
+        .collect(Collectors.toMap(DataEntityQualityStatusDto::getStatus, identity()));
+
+    public static Optional<DataEntityQualityStatusDto> findByStatus(final String status) {
+        return Optional.ofNullable(MAP.get(status));
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityTaskRunStatusDto.java

@@ -0,0 +1,36 @@
+package org.opendatadiscovery.oddplatform.dto;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import lombok.Getter;
+
+import static java.util.function.Function.identity;
+
+@Getter
+public enum DataEntityTaskRunStatusDto {
+    SUCCESS(1, "SUCCESS"),
+    FAILED(2, "FAILED"),
+    SKIPPED(3, "SKIPPED"),
+    BROKEN(4, "BROKEN"),
+    ABORTED(5, "ABORTED"),
+    UNKNOWN(6, "UNKNOWN");
+
+    private final short id;
+    private final String status;
+
+    DataEntityTaskRunStatusDto(final int id,
+                               final String status) {
+        this.id = (short) id;
+        this.status = status;
+    }
+
+    private static final Map<String, DataEntityTaskRunStatusDto> MAP = Arrays
+        .stream(DataEntityTaskRunStatusDto.values())
+        .collect(Collectors.toMap(DataEntityTaskRunStatusDto::getStatus, identity()));
+
+    public static Optional<DataEntityTaskRunStatusDto> findByStatus(final String status) {
+        return Optional.ofNullable(MAP.get(status));
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java

@@ -9,6 +9,8 @@ public enum FacetType {
     TAGS,
     GROUPS,
     STATUSES,
+    LAST_RUN_STATUSES,
+    DATA_QUALITY_RELATION,
     DATA_ENTITY;
 
     public static FacetType lookup(final String facetType) {

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapper.java

@@ -30,4 +30,11 @@ public interface FacetStateMapper {
     TermFacetState mapDto(final FacetStateDto state);
 
     FacetState mapDto(final List<CountableSearchFilter> entityClasses, final FacetStateDto state);
+
+    SearchFormData mapToFormData(final List<Long> namespaceIds,
+                                 final List<Long> datasourceIds,
+                                 final List<Long> ownerIds,
+                                 final List<Long> tagIds,
+                                 final List<Integer> entityClasses,
+                                 final List<Integer> types);
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java

@@ -3,13 +3,15 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.UUID;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.jooq.JSONB;
 import org.opendatadiscovery.oddplatform.api.contract.model.CountableSearchFilter;
@@ -23,6 +25,8 @@
 import org.opendatadiscovery.oddplatform.api.contract.model.TermFacetState;
 import org.opendatadiscovery.oddplatform.api.contract.model.TermSearchFormData;
 import org.opendatadiscovery.oddplatform.api.contract.model.TermSearchFormDataFilters;
+import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto;
+import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto;
 import org.opendatadiscovery.oddplatform.dto.FacetStateDto;
 import org.opendatadiscovery.oddplatform.dto.FacetType;
 import org.opendatadiscovery.oddplatform.dto.SearchFilterDto;
@@ -45,7 +49,9 @@ public class FacetStateMapperImpl implements FacetStateMapper {
             SearchFormDataFilters::getOwners, FacetType.OWNERS,
             SearchFormDataFilters::getTags, FacetType.TAGS,
             SearchFormDataFilters::getGroups, FacetType.GROUPS,
-            SearchFormDataFilters::getStatuses, FacetType.STATUSES
+            SearchFormDataFilters::getStatuses, FacetType.STATUSES,
+            SearchFormDataFilters::getLastRunStatuses, FacetType.LAST_RUN_STATUSES,
+            SearchFormDataFilters::getDataQualityRelations, FacetType.DATA_QUALITY_RELATION
         );
 
     private static final Map<Function<TermSearchFormDataFilters, List<SearchFilterState>>, FacetType>
@@ -170,7 +176,69 @@ public FacetState mapDto(final List<CountableSearchFilter> entityClasses, final
             .owners(getSearchFiltersForFacetType(state, FacetType.OWNERS))
             .namespaces(getSearchFiltersForFacetType(state, FacetType.NAMESPACES))
             .tags(getSearchFiltersForFacetType(state, FacetType.TAGS))
-            .groups(getSearchFiltersForFacetType(state, FacetType.GROUPS));
+            .groups(getSearchFiltersForFacetType(state, FacetType.GROUPS))
+            .statuses(getSearchFiltersForFacetType(state, FacetType.STATUSES))
+            .lastRunStatuses(getSearchFiltersForFacetType(state, FacetType.LAST_RUN_STATUSES))
+            .dataQualityRelations(getSearchFiltersForFacetType(state, FacetType.DATA_QUALITY_RELATION));
+    }
+
+    @Override
+    public SearchFormData mapToFormData(final List<Long> namespaceIds,
+                                        final List<Long> datasourceIds,
+                                        final List<Long> ownerIds,
+                                        final List<Long> tagIds,
+                                        final List<Integer> entityClasses,
+                                        final List<Integer> types) {
+        final SearchFormDataFilters filters = new SearchFormDataFilters()
+            .namespaces(getFilterStateList(namespaceIds, FacetType.NAMESPACES))
+            .datasources(getFilterStateList(datasourceIds, FacetType.DATA_SOURCES))
+            .owners(getFilterStateList(ownerIds, FacetType.OWNERS))
+            .tags(getFilterStateList(tagIds, FacetType.TAGS))
+            .entityClasses(getFilterStateListForEntityClasses(entityClasses))
+            .types(getFilterStateListForType(types));
+
+        return new SearchFormData().filters(filters).query("");
+    }
+
+    private List<SearchFilterState> getFilterStateList(final List<Long> filterIds, final FacetType filterType) {
+        if (CollectionUtils.isEmpty(filterIds)) {
+            return Collections.emptyList();
+        }
+
+        return filterIds.stream()
+            .map(id -> new SearchFilterState()
+                .entityId(id)
+                .entityName(filterType.name())
+                .selected(true))
+            .collect(Collectors.toList());
+    }
+
+    private List<SearchFilterState> getFilterStateListForEntityClasses(final List<Integer> filterIds) {
+        if (CollectionUtils.isEmpty(filterIds)) {
+            return Collections.emptyList();
+        }
+
+        return filterIds.stream()
+            .map(id -> new SearchFilterState()
+                .entityId(Long.valueOf(id))
+                .entityName(DataEntityClassDto.findById(id).orElseThrow(() -> new IllegalArgumentException(
+                    "Unknown data entity class id: %d".formatted(id))).name())
+                .selected(true))
+            .collect(Collectors.toList());
+    }
+
+    private List<SearchFilterState> getFilterStateListForType(final List<Integer> filterIds) {
+        if (CollectionUtils.isEmpty(filterIds)) {
+            return Collections.emptyList();
+        }
+
+        return filterIds.stream()
+            .map(id -> new SearchFilterState()
+                .entityId(Long.valueOf(id))
+                .entityName(DataEntityTypeDto.findById(id).orElseThrow(() -> new IllegalArgumentException(
+                    "Unknown data entity type id: %d".formatted(id))).name())
+                .selected(true))
+            .collect(Collectors.toList());
     }
 
     private SearchFilterDto mapFilter(final SearchFilterState f, final FacetType type) {

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataEntityRepositoryImpl.java

@@ -439,6 +439,7 @@ public Mono<DataEntityPojo> setInternalDescription(final long dataEntityId, fina
 
     @Override
     public Mono<Long> countByState(final FacetStateDto state, final OwnerPojo owner) {
+//      count can be different from findByState result, because ENTITY_CLASSES are ignored during countByState
         final List<Condition> conditions = new ArrayList<>(jooqFTSHelper
             .facetStateConditions(state, DATA_ENTITY_CONDITIONS, List.of(FacetType.ENTITY_CLASSES)));
         if (!deletedEntitiesAreRequested(state.getState())) {

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRunsRepositoryImpl.java

@@ -46,6 +46,7 @@ public class ReactiveDataQualityRunsRepositoryImpl implements ReactiveDataQualit
     public static final String DATA_QUALITY_TEST_TYPE =
         "specific_attributes->'DATA_QUALITY_TEST'->'expectation'->>'category'";
     public static final String ID = "id";
+    public static final String DATA_ENTITY_ID = "data_entity_id";
     public static final String ODDRN = "oddrn";
     public static final String CATEGORY = "category";
     public static final String CATEGORIES_CTE = "categories_cte";
@@ -63,30 +64,9 @@ public class ReactiveDataQualityRunsRepositoryImpl implements ReactiveDataQualit
 
     @Override
     public Flux<DataQualityRunsRecord> getLatestDataQualityRunsResults(final DataQualityTestFiltersDto filtersDto) {
-        final Table<Record2<Long, String>> testFilters = generateTestFiltersCte(filtersDto);
-        final Select<Record3<Long, String, String>> deCategoryCTE =
-                DSL.select(DATA_ENTITY.ID.as(ID), DATA_ENTITY.ODDRN.as(ODDRN),
-                                field(DATA_QUALITY_TEST_TYPE, String.class).as(CATEGORY))
-                        .from(DATA_ENTITY, testFilters)
-                        .where(DATA_ENTITY.TYPE_ID.eq(DataEntityTypeDto.JOB.getId())
-                                .and(field(DATA_QUALITY_TEST_TYPE, String.class).isNotNull()))
-                        .and(testFilters.field(DATA_ENTITY.ID).eq(DATA_ENTITY.ID));
-
-        final Table<Record3<Long, String, String>> categoriesSubTable = deCategoryCTE.asTable(CATEGORIES_CTE);
+        final Table<Record3<Long, String, String>> categoriesSubTable = getCTEForLastRuns(filtersDto);
         final List<TableLike<?>> fromList = new ArrayList<>(List.of(categoriesSubTable, DATA_ENTITY_TASK_LAST_RUN));
-        final List<Condition> conditionList
-                = new ArrayList<>(List.of(categoriesSubTable.field(ODDRN, String.class)
-                    .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN)));
-
-        if (shouldAddFiltersForDataEntity(filtersDto)) {
-            final Table<Record2<Long, String>> deFilters = generateDataEntityFiltersCte(filtersDto);
-
-            fromList.addAll(List.of(deFilters, DATA_QUALITY_TEST_RELATIONS));
-
-            conditionList.add(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN
-                    .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN));
-            conditionList.add(deFilters.field(DATA_ENTITY.ODDRN).eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN));
-        }
+        final List<Condition> conditionList = getConditionsForLastRuns(filtersDto, fromList, categoriesSubTable);
 
         final SelectSeekStep1<Record3<String, String, Integer>, String> query
                 = DSL.select(
@@ -195,6 +175,39 @@ public Flux<MonitoredtablesRecord> getMonitoredTables(final DataQualityTestFilte
             .map(item -> item.into(MonitoredtablesRecord.class));
     }
 
+    private Table<Record3<Long, String, String>> getCTEForLastRuns(final DataQualityTestFiltersDto filtersDto) {
+        final Table<Record2<Long, String>> testFilters = generateTestFiltersCte(filtersDto);
+        final Select<Record3<Long, String, String>> deCategoryCTE =
+            DSL.select(DATA_ENTITY.ID.as(ID), DATA_ENTITY.ODDRN.as(ODDRN),
+                    field(DATA_QUALITY_TEST_TYPE, String.class).as(CATEGORY))
+                .from(DATA_ENTITY, testFilters)
+                .where(DATA_ENTITY.TYPE_ID.eq(DataEntityTypeDto.JOB.getId())
+                    .and(field(DATA_QUALITY_TEST_TYPE, String.class).isNotNull()))
+                .and(testFilters.field(DATA_ENTITY.ID).eq(DATA_ENTITY.ID));
+
+        return deCategoryCTE.asTable(CATEGORIES_CTE);
+    }
+
+    private List<Condition> getConditionsForLastRuns(final DataQualityTestFiltersDto filtersDto,
+                                                     final List<TableLike<?>> fromList,
+                                                     final Table<Record3<Long, String, String>> categoriesSubTable) {
+        final List<Condition> conditionList
+            = new ArrayList<>(List.of(categoriesSubTable.field(ODDRN, String.class)
+            .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN)));
+
+        if (shouldAddFiltersForDataEntity(filtersDto)) {
+            final Table<Record2<Long, String>> deFilters = generateDataEntityFiltersCte(filtersDto);
+
+            fromList.addAll(List.of(deFilters, DATA_QUALITY_TEST_RELATIONS));
+
+            conditionList.add(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN
+                .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN));
+            conditionList.add(deFilters.field(DATA_ENTITY.ODDRN).eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN));
+        }
+
+        return conditionList;
+    }
+
     private SelectConditionStep<Record> getDataQualityQuery(final Table<Record2<Long, String>> dataEntityCTE,
                                                             final DataQualityTestFiltersDto filtersDto) {
         final Table<Record2<Long, String>> testFilters = generateTestFiltersCte(filtersDto);

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java

@@ -51,4 +51,9 @@ Mono<Map<SearchFilterId, Long>> getStatusFacetForDataEntity(final String query,
                                                                 final int page,
                                                                 final int size,
                                                                 final FacetStateDto state);
+
+    Mono<Map<SearchFilterId, Long>> getLastRunStatusesFacetForDataEntity(final String query,
+                                                                         final int page,
+                                                                         final int size,
+                                                                         final FacetStateDto state);
 }