From 4610e18d4a239e2a7c33a083f97add5413597b3f Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Thu, 22 Dec 2022 23:52:33 +0530 Subject: [PATCH 01/14] feat: connect different schemas in registry by giving in reference while creating entity --- .../test/java/e2e/registry/inviteFlow.json | 5 +++-- .../test/java/e2e/registry/registry.feature | 5 ++++- .../registry/middleware/util/Constants.java | 1 + .../sunbirdrc/registry/dao/VertexWriter.java | 19 ++++++++++++++++--- .../registry/dao/VertexWriterTest.java | 2 +- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/java/apitest/src/test/java/e2e/registry/inviteFlow.json b/java/apitest/src/test/java/e2e/registry/inviteFlow.json index c8a0a1ea0..1d1a8b5eb 100644 --- a/java/apitest/src/test/java/e2e/registry/inviteFlow.json +++ b/java/apitest/src/test/java/e2e/registry/inviteFlow.json @@ -11,12 +11,13 @@ }, "instituteSchema": { "name": "Institute", - "schema": "{\n \"$schema\": \"http://json-schema.org/draft-07/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"Institute\": {\n \"$ref\": \"#/definitions/Institute\"\n }\n },\n \"required\": [\n \"Institute\"\n ],\n \"title\":\"Institute\",\n \"definitions\": {\n \"Institute\": {\n \"$id\": \"#/properties/Institute\",\n \"type\": \"object\",\n \"title\": \"The Institute Schema\",\n \"required\": [\n\n ],\n \"properties\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"email\": {\n \"type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"_osConfig\": {\n \"osComment\": [\"This section contains the OpenSABER specific configuration information\",\n \"privateFields: Optional; list of field names to be encrypted and stored in database\",\n \"signedFields: Optional; list of field names that must be pre-signed\",\n \"indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.\",\n \"uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name\",\n \"systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid\"],\n\n \"privateFields\": [\n\n ],\n \"signedFields\": [],\n \"indexFields\": [],\n \"uniqueIndexFields\": [],\n \"systemFields\": [\n \"_osCreatedAt\",\n \"_osUpdatedAt\",\n \"_osCreatedBy\",\n \"_osUpdatedBy\",\n \"_osAttestedData\",\n \"_osClaimId\",\n \"_osState\"\n ],\n \"inviteRoles\":[\"Board\"],\n \"enableLogin\": true,\n \"ownershipAttributes\": [\n {\n \"email\": \"/email\",\n \"mobile\": \"/mobile\",\n \"userId\": \"/mobile\"\n }\n ]\n }\n}", + "schema": "{\n \"$schema\": \"http://json-schema.org/draft-07/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"Institute\": {\n \"$ref\": \"#/definitions/Institute\"\n }\n },\n \"required\": [\n \"Institute\"\n ],\n \"title\":\"Institute\",\n \"definitions\": {\n \"Institute\": {\n \"$id\": \"#/properties/Institute\",\n \"type\": \"object\",\n \"title\": \"The Institute Schema\",\n \"required\": [\n\n ],\n \"properties\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"email\": {\n \"type\": \"string\"\n },\n \"references\": {\n \"type\": \"string\"\n}, \n \"mobile\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"_osConfig\": {\n \"osComment\": [\"This section contains the OpenSABER specific configuration information\",\n \"privateFields: Optional; list of field names to be encrypted and stored in database\",\n \"signedFields: Optional; list of field names that must be pre-signed\",\n \"indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.\",\n \"uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name\",\n \"systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid\"],\n\n \"privateFields\": [\n\n ],\n \"signedFields\": [],\n \"indexFields\": [],\n \"uniqueIndexFields\": [],\n \"systemFields\": [\n \"_osCreatedAt\",\n \"_osUpdatedAt\",\n \"_osCreatedBy\",\n \"_osUpdatedBy\",\n \"_osAttestedData\",\n \"_osClaimId\",\n \"_osState\"\n ],\n \"inviteRoles\":[\"Board\"],\n \"enableLogin\": true,\n \"ownershipAttributes\": [\n {\n \"email\": \"/email\",\n \"mobile\": \"/mobile\",\n \"userId\": \"/mobile\"\n }\n ]\n }\n}", "status": "PUBLISHED" }, "instituteRequest": { "name": "test_institute", "email": "test_institute@mail.com", - "mobile": "456" + "mobile": "456", + "references": "did:Board:123" } } diff --git a/java/apitest/src/test/java/e2e/registry/registry.feature b/java/apitest/src/test/java/e2e/registry/registry.feature index 72fcda46c..3bee5a64e 100644 --- a/java/apitest/src/test/java/e2e/registry/registry.feature +++ b/java/apitest/src/test/java/e2e/registry/registry.feature @@ -274,6 +274,7 @@ Feature: Registry api tests And request sample.boardInviteRequest When method post Then status 200 + * def boardOsid = response.result.Board.osid # get board token * url authUrl * path 'auth/realms/sunbird-rc/protocol/openid-connect/token' @@ -299,7 +300,9 @@ Feature: Registry api tests # invite institute with token Given url registryUrl And path 'api/v1/Institute/invite' - And request sample.instituteRequest + * def requestBody = sample.instituteRequest + * requestBody.references = 'did:Board:' + boardOsid + And request requestBody And header Authorization = board_token When method post Then status 200 diff --git a/java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java b/java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java index a6d1cbf8a..0d4e7230f 100644 --- a/java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java +++ b/java/middleware-commons/src/main/java/dev/sunbirdrc/registry/middleware/util/Constants.java @@ -125,6 +125,7 @@ public class Constants { public static final String MOBILE = "mobile"; public static final String SVG_MEDIA_TYPE = "image/svg+xml"; public static final String CONNECTION_FAILURE = "CONNECTION_FAILURE"; + public static final String DID_TYPE = "did"; public enum GraphDatabaseProvider { NEO4J("NEO4J"), ORIENTDB("ORIENTDB"), SQLG("SQLG"), CASSANDRA("CASSANDRA"), TINKERGRAPH("TINKERGRAPH"); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java index f9eada681..d3c2d2b09 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.Map; +import static dev.sunbirdrc.registry.middleware.util.Constants.DID_TYPE; + /** * Helps in writing a vertex, edge into the database */ @@ -31,7 +33,7 @@ public class VertexWriter { private DatabaseProvider databaseProvider; private String parentOSid; private static final String EMPTY_STR = ""; - + private static final String DID_SEPERATOR = ":"; private Logger logger = LoggerFactory.getLogger(VertexWriter.class); public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName) { @@ -215,8 +217,9 @@ private Vertex processNode(String label, JsonNode jsonObject) { jsonObject.fields().forEachRemaining(entry -> { JsonNode entryValue = entry.getValue(); logger.debug("Processing {} -> {}", entry.getKey(), entry.getValue()); - - if (entryValue.isValueNode()) { + if (entryValue.isValueNode() && entryValue.asText().startsWith(DID_TYPE)) { + updateParentForReferencingNode(vertex, entry, entryValue); + } else if (entryValue.isValueNode()) { // Directly add under the vertex as a property vertex.property(entry.getKey(), ValueType.getValue(entryValue)); } else if (entryValue.isObject()) { @@ -229,6 +232,16 @@ private Vertex processNode(String label, JsonNode jsonObject) { return vertex; } + private void updateParentForReferencingNode(Vertex vertex, Map.Entry entry, JsonNode entryValue) { + String[] dids = entryValue.asText().split(DID_SEPERATOR); + Iterator vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, dids[2]); + while(vertexIterator.hasNext()) { + Vertex dependent = vertexIterator.next(); + addEdge(entry.getKey(), vertex, dependent); + vertex.property(RefLabelHelper.getLabel(entry.getKey(), uuidPropertyName), dids[2]); + } + } + /** * Adds an edge between two vertices * diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java index 0b95b0c1b..a41a59888 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java @@ -90,7 +90,7 @@ public void addEdge() { @Test public void writeNodeEntity() { - String recordStr = "{\"entityName\": {\"a\":\"b\", \"cObj\": {\"d\":\"e\"}, \"fArr\": [\"i1\", \"i2\"], \"gObjArr\": [{\"i1\": \"v1\"}, {\"i2\":\"v2\"}]}}"; + String recordStr = "{\"entityName\": {\"ref\": \"did:anotherEntity:1234\", \"a\":\"b\", \"cObj\": {\"d\":\"e\"}, \"fArr\": [\"i1\", \"i2\"], \"gObjArr\": [{\"i1\": \"v1\"}, {\"i2\":\"v2\"}]}}"; JsonNode recordNode = null; try { recordNode = new ObjectMapper().readTree(recordStr); From e9d5dc7e76979ad3274b7f3dcdc76465eed2b1d8 Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Fri, 23 Dec 2022 20:22:18 +0530 Subject: [PATCH 02/14] feat: add unit test cases for VertexWriter --- .../registry/dao/VertexWriterTest.java | 104 +++++++++++++++--- 1 file changed, 91 insertions(+), 13 deletions(-) diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java index a41a59888..edb358e9b 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java @@ -3,19 +3,26 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import dev.sunbirdrc.registry.middleware.util.Constants; import dev.sunbirdrc.registry.model.DBConnectionInfo; import dev.sunbirdrc.registry.model.DBConnectionInfoMgr; import dev.sunbirdrc.registry.sink.DBProviderFactory; import dev.sunbirdrc.registry.sink.DatabaseProvider; import dev.sunbirdrc.registry.sink.OSGraph; -import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.env.Environment; @@ -23,6 +30,11 @@ import org.springframework.test.context.junit4.SpringRunner; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.*; @RunWith(SpringRunner.class) @@ -42,20 +54,51 @@ public class VertexWriterTest { private static final String testUuidPropertyName = "tid"; private VertexWriter vertexWriter; + Vertex vertex; @Before public void setUp() throws Exception { dbConnectionInfoMgr.setUuidPropertyName(testUuidPropertyName); - mockDatabaseProvider = dbProviderFactory.getInstance(null); - try (OSGraph osGraph = mockDatabaseProvider.getOSGraph()) { - graph = osGraph.getGraphStore(); - vertexWriter = new VertexWriter(graph, mockDatabaseProvider, testUuidPropertyName); - } + mockDatabaseProvider = Mockito.mock(DatabaseProvider.class); + graph = Mockito.mock(Graph.class); + OSGraph osGraph = Mockito.mock(OSGraph.class); + Mockito.when(mockDatabaseProvider.getOSGraph()).thenReturn(osGraph); + Mockito.when(osGraph.getGraphStore()).thenReturn(graph); + vertex = Mockito.mock(Vertex.class); + Mockito.when(graph.addVertex(anyString())).thenReturn(vertex); + vertexWriter = new VertexWriter(graph, mockDatabaseProvider, testUuidPropertyName); } @Test - public void ensureParentVertex() { + public void ensureParentVertexWhenParentIndexAlreadyExists() { + String parentLabel = "Test_Group"; + GraphTraversalSource graphTraversalSource = Mockito.mock(GraphTraversalSource.class); + GraphTraversal graphTraversal = Mockito.mock(GraphTraversal.class); + Mockito.when(graph.traversal()).thenReturn(graphTraversalSource); + Mockito.when(graphTraversalSource.clone()).thenReturn(graphTraversalSource); + Mockito.when(graphTraversalSource.V()).thenReturn(graphTraversal); + Mockito.when(graphTraversal.hasLabel(P.eq(parentLabel))).thenReturn(graphTraversal); + Mockito.when(graphTraversal.hasNext()).thenReturn(true); + Mockito.when(graphTraversal.next()).thenReturn(vertex); + Vertex actualVertex = vertexWriter.ensureParentVertex(parentLabel); + assertEquals(vertex, actualVertex); + } + @Test + public void ensureParentVertexWhenParentIndexDoesNotExist() { + String parentLabel = "Test_Group"; + GraphTraversalSource graphTraversalSource = Mockito.mock(GraphTraversalSource.class); + GraphTraversal graphTraversal = Mockito.mock(GraphTraversal.class); + Mockito.when(graph.traversal()).thenReturn(graphTraversalSource); + Mockito.when(graphTraversalSource.clone()).thenReturn(graphTraversalSource); + Mockito.when(graphTraversalSource.V()).thenReturn(graphTraversal); + Mockito.when(graphTraversal.hasLabel(P.eq(parentLabel))).thenReturn(graphTraversal); + Mockito.when(graphTraversal.hasNext()).thenReturn(false); + Mockito.when(vertex.id()).thenReturn("123"); + Vertex actualVertex = vertexWriter.ensureParentVertex(parentLabel); + assertEquals(vertex, actualVertex); + Mockito.verify(actualVertex, Mockito.times(1)).property(Constants.INDEX_FIELDS, ""); + Mockito.verify(actualVertex, Mockito.times(1)).property(Constants.UNIQUE_INDEX_FIELDS, ""); } private Vertex createVertexImpl(String lblStr) { @@ -65,15 +108,25 @@ private Vertex createVertexImpl(String lblStr) { @Test public void createVertex() { String lblStr = "LabelStr1"; + Mockito.when(mockDatabaseProvider.generateId(vertex)).thenReturn("123"); Vertex vertexCreated = createVertexImpl(lblStr); - Assert.assertTrue(vertexCreated != null && - vertexCreated.label().equals(lblStr) && - vertexCreated.value(testUuidPropertyName) != null && - vertexCreated.value(Constants.TYPE_STR_JSON_LD) != null); + Mockito.verify(vertexCreated, Mockito.times(1)).property("@type", lblStr); + Mockito.verify(vertexCreated, Mockito.times(1)).property(testUuidPropertyName, "123"); } @Test public void writeSingleNode() { + Vertex parentVertex = Mockito.mock(Vertex.class); + String label = "dummy_lbl"; + ObjectNode entryValue = JsonNodeFactory.instance.objectNode(); + TextNode value = JsonNodeFactory.instance.textNode("value1"); + entryValue.set("field1", value); + Mockito.when(mockDatabaseProvider.getId(vertex)).thenReturn("123"); + Vertex actualVertex = vertexWriter.writeSingleNode(parentVertex, label, entryValue); + + Mockito.verify(parentVertex, Mockito.times(1)).addEdge(label, vertex); + Mockito.verify(parentVertex, Mockito.times(1)).property(label + "_" + testUuidPropertyName, "123"); + assertEquals(vertex, actualVertex); } @Test @@ -84,8 +137,18 @@ public void addEdge() { vertexWriter.addEdge(eLabel, v1, v2); - Assert.assertTrue(v1.vertices(Direction.OUT).next() != null && - !v2.vertices(Direction.OUT).hasNext()); + Mockito.verify(v1, Mockito.times(1)).addEdge(eLabel, v2); + } + + @Test + public void test_shouldUpdateParentIndexProperty() { + List indexFields = new ArrayList<>(); + indexFields.add("name"); + indexFields.add("rollNo"); + Vertex parentVertex = Mockito.mock(Vertex.class); + String propertyName = "test"; + vertexWriter.updateParentIndexProperty(parentVertex, propertyName, indexFields); + Mockito.verify(parentVertex, Mockito.times(1)).property(propertyName, "name,rollNo"); } @Test @@ -97,7 +160,22 @@ public void writeNodeEntity() { } catch (IOException e) { e.printStackTrace(); } + GraphTraversalSource graphTraversalSource = Mockito.mock(GraphTraversalSource.class); + GraphTraversal graphTraversal = Mockito.mock(GraphTraversal.class); + VertexProperty vertexProperty = Mockito.mock(VertexProperty.class); + Mockito.when(mockDatabaseProvider.generateId(any())).thenReturn("123"); + Mockito.when(mockDatabaseProvider.getId(vertex)).thenReturn("123"); + Mockito.when(graph.traversal()).thenReturn(graphTraversalSource); + Mockito.when(graphTraversalSource.clone()).thenReturn(graphTraversalSource); + Mockito.when(graphTraversalSource.V()).thenReturn(graphTraversal); + Mockito.when(graphTraversal.hasLabel("anotherEntity")).thenReturn(graphTraversal); + Mockito.when(graphTraversal.has(testUuidPropertyName, "1234")).thenReturn(graphTraversal); + Mockito.when(graphTraversal.hasNext()).thenReturn(true).thenReturn(false); + Mockito.when(graphTraversal.next()).thenReturn(vertex); + Mockito.when(vertexProperty.isPresent()).thenReturn(false); + Mockito.when(vertex.property(anyString())).thenReturn(vertexProperty); String id = vertexWriter.writeNodeEntity(recordNode); Assert.assertTrue(id != null); + assertEquals("123", id); } } \ No newline at end of file From 18afe7d7051a592beac4db9664a549d5b7acf1f4 Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Thu, 12 Jan 2023 18:46:04 +0530 Subject: [PATCH 03/14] expand child object in elastic search service --- Makefile | 9 +++ .../service/ElasticSearchService.java | 61 ++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b76eac927..c9a549179 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,15 @@ test: @cd java/apitest && MODE=async ../mvnw -Pe2e test || echo 'Tests failed' @docker-compose down @rm -rf db-data-4 || echo "no permission to delete" + # test with elastic search service + @RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-5 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.ElasticSearchService docker-compose up -d db keycloak registry certificate-signer certificate-api + @echo "Starting the test" && sh build/wait_for_port.sh 8080 + @echo "Starting the test" && sh build/wait_for_port.sh 8081 + @docker-compose ps + @curl -v http://localhost:8081/health + @cd java/apitest && ../mvnw -Pe2e test || echo 'Tests failed' + @docker-compose down + @rm -rf db-data-5 || echo "no permission to delete" make -C services/certificate-signer test make -C services/public-key-service test make -C services/context-proxy-service test diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java index 726c66d9d..56bf05b46 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java @@ -4,6 +4,11 @@ import java.util.ArrayList; import java.util.List; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -23,6 +28,8 @@ import dev.sunbirdrc.registry.middleware.util.Constants; import dev.sunbirdrc.registry.util.RecordIdentifier; +import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_TYPE; + /** * This class provide search option with Elastic search Hits elastic search * database to operate @@ -56,6 +63,9 @@ public class ElasticSearchService implements ISearchService { @Value("${audit.frame.suffix}") private String auditSuffix; + @Autowired + ObjectMapper objectMapper; + @Override public JsonNode search(JsonNode inputQueryNode) throws IOException { logger.debug("search request body = " + inputQueryNode); @@ -80,7 +90,8 @@ public JsonNode search(JsonNode inputQueryNode) throws IOException { ObjectNode resultNode = JsonNodeFactory.instance.objectNode(); for(String indexName : searchQuery.getEntityTypes()){ try{ - JsonNode node = elasticService.search(indexName.toLowerCase(), searchQuery); + JsonNode node = elasticService.search(indexName.toLowerCase(), searchQuery); + node = expandReference(node); resultNode.set(indexName, node); } catch (Exception e) { @@ -99,7 +110,53 @@ public JsonNode search(JsonNode inputQueryNode) throws IOException { } - private void updateStatusFilter(SearchQuery searchQuery) { + private ArrayNode expandReference(JsonNode node) { + ArrayNode arrayNode = (ArrayNode) node; + ArrayNode ans = JsonNodeFactory.instance.arrayNode(); + for (JsonNode node1 : arrayNode) { + ObjectNode objectNode = (ObjectNode) node1; + List removableReferenceKeys = new ArrayList<>(); + objectNode.fields().forEachRemaining(objectField -> { + if(objectField.getValue().asText().contains("did:")) { + String[] splits = objectField.getValue().asText().split(":"); + String indexName = splits[1].toLowerCase(); + String osid = splits[2]; + SearchQuery searchQuery1 = null; + ArrayNode node2 = null; + try { + searchQuery1 = getSearchQuery(splits, osid); + node2 = (ArrayNode) elasticService.search(indexName, searchQuery1); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + if(node2.size() > 0) { + objectField.setValue(node2.get(0)); + } else { + removableReferenceKeys.add(objectField.getKey()); + } + } + }); + for (String referenceKeys: removableReferenceKeys) { + objectNode.remove(referenceKeys); + } + ans.add(objectNode); + } + return ans; + } + + private SearchQuery getSearchQuery(String[] splits, String osid) throws JsonProcessingException { + String filter = "{\"filters\": {\"osid\":{ \"eq\":\"" + osid + "\"}}}"; + ObjectNode jsonNode = (ObjectNode) objectMapper.readTree(filter); + ArrayNode entity = JsonNodeFactory.instance.arrayNode(); + entity.add(splits[1]); + jsonNode.set(ENTITY_TYPE, entity); + SearchQuery searchQuery1 = getSearchQuery(jsonNode, offset, limit); + return searchQuery1; + } + + private void updateStatusFilter(SearchQuery searchQuery) { List filterList = searchQuery.getFilters(); Filter filter = new Filter(Constants.STATUS_KEYWORD, FilterOperators.neq, Constants.STATUS_INACTIVE); filterList.add(filter); From e3c12dab21b543d251572134371e79b56f28d916 Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Mon, 16 Jan 2023 15:56:46 +0530 Subject: [PATCH 04/14] add a flag for expanding reference object --- .../registry/dao/RegistryDaoImpl.java | 8 ++-- .../sunbirdrc/registry/dao/VertexWriter.java | 7 ++- .../service/ElasticSearchService.java | 43 +++++++++++-------- .../registry/service/NativeReadService.java | 5 ++- .../registry/service/NativeSearchService.java | 5 ++- .../registry/service/impl/AuditDBWriter.java | 7 +-- .../service/impl/RegistryServiceImpl.java | 12 +++--- .../registry/util/EntityParenter.java | 6 ++- .../src/main/resources/application.yml | 4 +- .../registry/dao/VertexWriterTest.java | 5 ++- .../registry/dao/impl/SearchDaoImplTest.java | 9 ++-- .../service/impl/RegistryServiceImplTest.java | 12 +++--- 12 files changed, 73 insertions(+), 50 deletions(-) diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java index af470b5c0..2e701cd49 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java @@ -17,6 +17,7 @@ public class RegistryDaoImpl implements IRegistryDao { public String uuidPropertyName; + private final boolean expandReferenceObj; private IDefinitionsManager definitionsManager; private DatabaseProvider databaseProvider; private List privatePropertyList; @@ -33,10 +34,11 @@ public void setPrivatePropertyList(List privatePropertyList) { this.privatePropertyList = privatePropertyList; } - public RegistryDaoImpl(DatabaseProvider dbProvider, IDefinitionsManager defnManager, String uuidPropName) { + public RegistryDaoImpl(DatabaseProvider dbProvider, IDefinitionsManager defnManager, String uuidPropName, boolean expandReferenceObj) { databaseProvider = dbProvider; definitionsManager = defnManager; uuidPropertyName = uuidPropName; + this.expandReferenceObj = expandReferenceObj; } public DatabaseProvider getDatabaseProvider() { @@ -50,7 +52,7 @@ public DatabaseProvider getDatabaseProvider() { * @return */ public String addEntity(Graph graph, JsonNode rootNode) { - VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName); + VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName, expandReferenceObj); String entityId = vertexWriter.writeNodeEntity(rootNode); return entityId; } @@ -104,7 +106,7 @@ public void updateVertex(Graph graph, Vertex vertex, JsonNode inputJsonNode, Str if (databaseProvider.getId(vertex).equals(osidVal)) { updateObject(graph, vertex, (ObjectNode) inputJsonNode); } else { - VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName); + VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName, expandReferenceObj); if(inputJsonNode.get(uuidPropertyName) != null) { vertexWriter.writeSingleNode(vertex, objectName, inputJsonNode.get(objectName)); } else { diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java index d3c2d2b09..6103110d5 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java @@ -16,6 +16,7 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import java.util.ArrayList; import java.util.Iterator; @@ -35,11 +36,13 @@ public class VertexWriter { private static final String EMPTY_STR = ""; private static final String DID_SEPERATOR = ":"; private Logger logger = LoggerFactory.getLogger(VertexWriter.class); + private boolean expandReferenceObj; - public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName) { + public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName, boolean expandReferenceObj) { this.graph = graph; this.databaseProvider = databaseProvider; this.uuidPropertyName = uuidPropertyName; + this.expandReferenceObj = expandReferenceObj; } /** @@ -217,7 +220,7 @@ private Vertex processNode(String label, JsonNode jsonObject) { jsonObject.fields().forEachRemaining(entry -> { JsonNode entryValue = entry.getValue(); logger.debug("Processing {} -> {}", entry.getKey(), entry.getValue()); - if (entryValue.isValueNode() && entryValue.asText().startsWith(DID_TYPE)) { + if (entryValue.isValueNode() && entryValue.asText().startsWith(DID_TYPE) && expandReferenceObj) { updateParentForReferencingNode(vertex, entry, entryValue); } else if (entryValue.isValueNode()) { // Directly add under the vertex as a property diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java index 56bf05b46..9ad87d6f6 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java @@ -63,6 +63,9 @@ public class ElasticSearchService implements ISearchService { @Value("${audit.frame.suffix}") private String auditSuffix; + @Value("${registry.expandReference}") + private boolean expandReferenceObj; + @Autowired ObjectMapper objectMapper; @@ -90,9 +93,11 @@ public JsonNode search(JsonNode inputQueryNode) throws IOException { ObjectNode resultNode = JsonNodeFactory.instance.objectNode(); for(String indexName : searchQuery.getEntityTypes()){ try{ - JsonNode node = elasticService.search(indexName.toLowerCase(), searchQuery); - node = expandReference(node); - resultNode.set(indexName, node); + JsonNode searchedNode = elasticService.search(indexName.toLowerCase(), searchQuery); + if(expandReferenceObj) { + searchedNode = expandReference(searchedNode); + } + resultNode.set(indexName, searchedNode); } catch (Exception e) { logger.error("Elastic search operation - {}", e); @@ -110,29 +115,29 @@ public JsonNode search(JsonNode inputQueryNode) throws IOException { } - private ArrayNode expandReference(JsonNode node) { - ArrayNode arrayNode = (ArrayNode) node; - ArrayNode ans = JsonNodeFactory.instance.arrayNode(); - for (JsonNode node1 : arrayNode) { - ObjectNode objectNode = (ObjectNode) node1; + private ArrayNode expandReference(JsonNode searchedNode) { + ArrayNode arrayNode = (ArrayNode) searchedNode; + ArrayNode nodeWithExpandedReference = JsonNodeFactory.instance.arrayNode(); + for (JsonNode node : arrayNode) { + ObjectNode objectNode = (ObjectNode) node; List removableReferenceKeys = new ArrayList<>(); objectNode.fields().forEachRemaining(objectField -> { - if(objectField.getValue().asText().contains("did:")) { + if(objectField.getValue().asText().startsWith("did:")) { String[] splits = objectField.getValue().asText().split(":"); String indexName = splits[1].toLowerCase(); String osid = splits[2]; - SearchQuery searchQuery1 = null; - ArrayNode node2 = null; + SearchQuery searchQuery = null; + ArrayNode referenceNode = null; try { - searchQuery1 = getSearchQuery(splits, osid); - node2 = (ArrayNode) elasticService.search(indexName, searchQuery1); + searchQuery = getSearchQuery(splits[1], osid); + referenceNode = (ArrayNode) elasticService.search(indexName, searchQuery); } catch (JsonProcessingException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } - if(node2.size() > 0) { - objectField.setValue(node2.get(0)); + if(referenceNode.size() > 0) { + objectField.setValue(referenceNode.get(0)); } else { removableReferenceKeys.add(objectField.getKey()); } @@ -141,16 +146,16 @@ private ArrayNode expandReference(JsonNode node) { for (String referenceKeys: removableReferenceKeys) { objectNode.remove(referenceKeys); } - ans.add(objectNode); + nodeWithExpandedReference.add(objectNode); } - return ans; + return nodeWithExpandedReference; } - private SearchQuery getSearchQuery(String[] splits, String osid) throws JsonProcessingException { + private SearchQuery getSearchQuery(String entityName, String osid) throws JsonProcessingException { String filter = "{\"filters\": {\"osid\":{ \"eq\":\"" + osid + "\"}}}"; ObjectNode jsonNode = (ObjectNode) objectMapper.readTree(filter); ArrayNode entity = JsonNodeFactory.instance.arrayNode(); - entity.add(splits[1]); + entity.add(entityName); jsonNode.set(ENTITY_TYPE, entity); SearchQuery searchQuery1 = getSearchQuery(jsonNode, offset, limit); return searchQuery1; diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeReadService.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeReadService.java index feffa2a8e..7bc647770 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeReadService.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeReadService.java @@ -47,6 +47,9 @@ public class NativeReadService implements IReadService { @Value("${audit.enabled}") private boolean auditEnabled; + @Value("${registry.expandReference}") + private boolean expandReferenceObj; + /** * This method interacts with the native db and reads the record * @@ -59,7 +62,7 @@ public class NativeReadService implements IReadService { @Override public JsonNode getEntity(Shard shard, String userId, String id, String entityType, ReadConfigurator configurator) throws Exception { DatabaseProvider dbProvider = shard.getDatabaseProvider(); - IRegistryDao registryDao = new RegistryDaoImpl(dbProvider, definitionsManager, uuidPropertyName); + IRegistryDao registryDao = new RegistryDaoImpl(dbProvider, definitionsManager, uuidPropertyName, expandReferenceObj); try (OSGraph osGraph = dbProvider.getOSGraph()) { Graph graph = osGraph.getGraphStore(); try (Transaction tx = dbProvider.startTransaction(graph)) { diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeSearchService.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeSearchService.java index 67c3452e6..3961e4754 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeSearchService.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/NativeSearchService.java @@ -27,7 +27,6 @@ import dev.sunbirdrc.registry.dao.IRegistryDao; import dev.sunbirdrc.registry.dao.RegistryDaoImpl; import dev.sunbirdrc.registry.dao.SearchDaoImpl; -import dev.sunbirdrc.registry.middleware.util.Constants; import dev.sunbirdrc.registry.middleware.util.JSONUtil; import dev.sunbirdrc.registry.model.DBConnectionInfo; import dev.sunbirdrc.registry.model.DBConnectionInfoMgr; @@ -78,6 +77,8 @@ public class NativeSearchService implements ISearchService { @Value("${search.expandInternal}") private boolean expandInternal; + @Value("${registry.expandReference}") + private boolean expandReferenceObj; @Override public JsonNode search(JsonNode inputQueryNode) throws IOException { @@ -109,7 +110,7 @@ public JsonNode search(JsonNode inputQueryNode) throws IOException { List transaction = new LinkedList<>(); Shard shard = shardManager.activateShard(dbConnection.getShardId()); - IRegistryDao registryDao = new RegistryDaoImpl(shard.getDatabaseProvider(), definitionsManager, uuidPropertyName); + IRegistryDao registryDao = new RegistryDaoImpl(shard.getDatabaseProvider(), definitionsManager, uuidPropertyName, expandReferenceObj); SearchDaoImpl searchDao = new SearchDaoImpl(registryDao); try (OSGraph osGraph = shard.getDatabaseProvider().getOSGraph()) { Graph graph = osGraph.getGraphStore(); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/AuditDBWriter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/AuditDBWriter.java index caf9ab52c..4f11d57e7 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/AuditDBWriter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/AuditDBWriter.java @@ -7,7 +7,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import com.fasterxml.jackson.databind.JsonNode; @@ -42,13 +41,15 @@ public class AuditDBWriter { @Autowired private EntityParenter entityParenter; + @Value("${registry.expandReference}") + private boolean expandReferenceObj; - public String auditToDB(Shard shard, JsonNode rootNode, String entityType) throws AuditFailedException { + public String auditToDB(Shard shard, JsonNode rootNode, String entityType) throws AuditFailedException { String entityId = "auditPlaceholderId"; Transaction tx = null; DatabaseProvider dbProvider = shard.getDatabaseProvider(); - IRegistryDao registryDao = new RegistryDaoImpl(dbProvider, definitionsManager, uuidPropertyName); + IRegistryDao registryDao = new RegistryDaoImpl(dbProvider, definitionsManager, uuidPropertyName, expandReferenceObj); try (OSGraph osGraph = dbProvider.getOSGraph()) { Graph graph = osGraph.getGraphStore(); tx = dbProvider.startTransaction(graph); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java index d1906f6d2..57925a933 100755 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java @@ -116,6 +116,8 @@ public class RegistryServiceImpl implements RegistryService { @Autowired private List healthIndicators; + @Value("${registry.expandReference}") + private boolean expandReferenceObj; public HealthCheckResponse health(Shard shard) throws Exception { HealthCheckResponse healthCheck; @@ -142,7 +144,7 @@ public HealthCheckResponse health(Shard shard) throws Exception { @Override public Vertex deleteEntityById(Shard shard, String userId, String uuid) throws Exception { DatabaseProvider databaseProvider = shard.getDatabaseProvider(); - IRegistryDao registryDao = new RegistryDaoImpl(databaseProvider, definitionsManager, uuidPropertyName); + IRegistryDao registryDao = new RegistryDaoImpl(databaseProvider, definitionsManager, uuidPropertyName, expandReferenceObj); try (OSGraph osGraph = databaseProvider.getOSGraph()) { Graph graph = osGraph.getGraphStore(); try (Transaction tx = databaseProvider.startTransaction(graph)) { @@ -202,7 +204,7 @@ public String addEntity(Shard shard, String userId, JsonNode rootNode, boolean s if (persistenceEnabled) { DatabaseProvider dbProvider = shard.getDatabaseProvider(); - IRegistryDao registryDao = new RegistryDaoImpl(dbProvider, definitionsManager, uuidPropertyName); + IRegistryDao registryDao = new RegistryDaoImpl(dbProvider, definitionsManager, uuidPropertyName, expandReferenceObj); try (OSGraph osGraph = dbProvider.getOSGraph()) { Graph graph = osGraph.getGraphStore(); tx = dbProvider.startTransaction(graph); @@ -268,7 +270,7 @@ public void updateEntity(Shard shard, String userId, String id, String jsonStrin } DatabaseProvider databaseProvider = shard.getDatabaseProvider(); - IRegistryDao registryDao = new RegistryDaoImpl(databaseProvider, definitionsManager, uuidPropertyName); + IRegistryDao registryDao = new RegistryDaoImpl(databaseProvider, definitionsManager, uuidPropertyName, expandReferenceObj); try (OSGraph osGraph = databaseProvider.getOSGraph()) { Graph graph = osGraph.getGraphStore(); try (Transaction tx = databaseProvider.startTransaction(graph)) { @@ -399,7 +401,7 @@ private void doUpdateArray(Shard shard, Graph graph, IRegistryDao registryDao, V Set updatedUuids = new HashSet(); Set previousArrayItemsUuids = vr.getArrayItemUuids(blankArrVertex); - VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName); + VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName, expandReferenceObj); for (JsonNode item : arrayNode) { if (item.isObject()) { @@ -491,7 +493,7 @@ private void doUpdate(Shard shard, Graph graph, IRegistryDao registryDao, Vertex // updateArrayItems one by one doUpdateArray(shard, graph, registryDao, vr, existArrayVertex, (ArrayNode) oneElementNode, userInputKey); } else { - VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName); + VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName, expandReferenceObj); vertexWriter.createArrayNode(rootVertex, oneElement.getKey(), (ArrayNode) oneElementNode); } registryDao.updateVertex(graph, existArrayVertex, oneElementNode, oneElement.getKey()); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java index 72e7269db..259844bf6 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java @@ -35,6 +35,8 @@ public class EntityParenter { @Value("${database.uuidPropertyName}") public String uuidPropertyName; + @Value("${registry.expandReference}") + public boolean expandReferenceObj; @Autowired private DBProviderFactory dbProviderFactory; @@ -135,7 +137,7 @@ public Optional ensureKnownParenters() { String parentLabel = ParentLabelGenerator.getLabel(defintionName); parentLabels.add(parentLabel); - VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName); + VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName, expandReferenceObj); Vertex v = vertexWriter.ensureParentVertex(parentLabel); ShardParentInfo shardParentInfo = new ShardParentInfo(defintionName, v); @@ -292,7 +294,7 @@ private void updateParentVertexIndexProperties(DatabaseProvider dbProvider, Vert Graph graph = osGraph.getGraphStore(); try (Transaction tx = dbProvider.startTransaction(graph)) { - VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName); + VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName, expandReferenceObj); Vertex v = graph.vertices(parentVertex.id()).next(); vertexWriter.updateParentIndexProperty(v, Constants.INDEX_FIELDS, indexFields); diff --git a/java/registry/src/main/resources/application.yml b/java/registry/src/main/resources/application.yml index 00a30a699..931b5e750 100644 --- a/java/registry/src/main/resources/application.yml +++ b/java/registry/src/main/resources/application.yml @@ -58,7 +58,7 @@ registry: redis: host: ${redis_host:localhost} port: ${redis_port:6379} - + expandReference: ${expand_reference:false} workflow: enabled: ${workflow.enable:true} @@ -331,7 +331,7 @@ registry: base: http://example.com/voc/teacher/1.0.0/ system: base: http://example.com/voc/opensaber/ - + expandReference: ${expand_reference:false} # This property is to be used for read request read: # The read mechanism to use, values could be either NativeReadService or ElasticReadService diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java index edb358e9b..89b2c324c 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java @@ -24,6 +24,7 @@ import org.junit.runner.RunWith; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.env.Environment; import org.springframework.test.context.ActiveProfiles; @@ -55,6 +56,8 @@ public class VertexWriterTest { private VertexWriter vertexWriter; Vertex vertex; + @Value("${registry.expandReference}") + private boolean expandReferenceObj; @Before public void setUp() throws Exception { @@ -66,7 +69,7 @@ public void setUp() throws Exception { Mockito.when(osGraph.getGraphStore()).thenReturn(graph); vertex = Mockito.mock(Vertex.class); Mockito.when(graph.addVertex(anyString())).thenReturn(vertex); - vertexWriter = new VertexWriter(graph, mockDatabaseProvider, testUuidPropertyName); + vertexWriter = new VertexWriter(graph, mockDatabaseProvider, testUuidPropertyName, expandReferenceObj); } @Test diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java index afc81504f..30f4d2900 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java @@ -33,6 +33,7 @@ import org.junit.runner.RunWith; import org.junit.runners.MethodSorters; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; @@ -58,8 +59,10 @@ public class SearchDaoImplTest { private final static int limit = 1; private final static boolean expandInternal = true; private List entities = new ArrayList<>(); + @Value("${registry.expandReference}") + private boolean expandReferenceObj; + - @Before public void initializeGraph() throws IOException { dbConnectionInfoMgr.setUuidPropertyName("tid"); @@ -67,7 +70,7 @@ public void initializeGraph() throws IOException { databaseProvider = dbProviderFactory.getInstance(null); graph = databaseProvider.getOSGraph().getGraphStore(); - IRegistryDao registryDao = new RegistryDaoImpl(databaseProvider, definitionsManager, "tid"); + IRegistryDao registryDao = new RegistryDaoImpl(databaseProvider, definitionsManager, "tid", expandReferenceObj); searchDao = new SearchDaoImpl(registryDao); populateGraph(); @@ -177,7 +180,7 @@ private SearchQuery getSearchQuery(List rootTypes, String property, Obje } private void populateGraph() { - VertexWriter vertexWriter = new VertexWriter(graph, databaseProvider, "tid"); + VertexWriter vertexWriter = new VertexWriter(graph, databaseProvider, "tid", expandReferenceObj); Vertex v1 = vertexWriter.createVertex("Teacher"); v1.property("serialNum", 1); v1.property("teacherName", "marko"); diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java index 9779b0f88..3c470988d 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java @@ -22,7 +22,6 @@ import dev.sunbirdrc.registry.service.SchemaService; import dev.sunbirdrc.registry.sink.DBProviderFactory; import dev.sunbirdrc.registry.sink.DatabaseProvider; -import dev.sunbirdrc.registry.sink.OSGraph; import dev.sunbirdrc.registry.sink.shard.Shard; import dev.sunbirdrc.registry.sink.shard.ShardManager; import dev.sunbirdrc.registry.util.*; @@ -31,11 +30,9 @@ import org.apache.commons.io.IOUtils; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.jupiter.api.AfterEach; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.*; @@ -52,7 +49,6 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.Arrays; -import java.util.Iterator; import static dev.sunbirdrc.registry.Constants.Schema; import static org.junit.Assert.*; @@ -66,6 +62,8 @@ public class RegistryServiceImplTest { @Value("${registry.schema.url}") private String schemaUrl; private String validationType = "json"; + @Value("${registry.expandReference}") + private boolean expandReferenceObj; public Constants.SchemaType getValidationType() throws IllegalArgumentException { String validationMechanism = validationType.toUpperCase(); @@ -166,7 +164,7 @@ public void initialize() throws IOException { } private void populateGraph() { - VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid"); + VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid", expandReferenceObj); Vertex v1 = vertexWriter.createVertex("Teacher"); v1.property("serialNum", 1); v1.property("teacherName", "marko"); @@ -380,14 +378,14 @@ public void shouldTestVertexWriter() throws Exception { } private String addStudentToGraph() throws JsonProcessingException { - VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid"); + VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid", expandReferenceObj); return vertexWriter.writeNodeEntity(objectMapper.readTree("{\"Student\": {\n" + " \"name\": \"abc\"\n" + "}}")); } private String addTeacherToGraph() throws JsonProcessingException { - VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid"); + VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid", expandReferenceObj); return vertexWriter.writeNodeEntity(objectMapper.readTree("{\"Teacher\": {\n" + " \"fullName\": \"abc\"\n" + "}}")); From a64a51ead7ff77866a1e815ea3fa2938b7d2c6ab Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Mon, 16 Jan 2023 16:15:25 +0530 Subject: [PATCH 05/14] fix e2e test case --- Makefile | 4 ++-- docker-compose.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c9a549179..857212c83 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ test: @docker-compose down @rm -rf db-data-2 || echo "no permission to delete" # test with native search service - @RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-3 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.NativeSearchService docker-compose up -d db keycloak registry certificate-signer certificate-api + @RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-3 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.NativeSearchService EXPAND_REFERENCE=false docker-compose up -d db keycloak registry certificate-signer certificate-api @echo "Starting the test" && sh build/wait_for_port.sh 8080 @echo "Starting the test" && sh build/wait_for_port.sh 8081 @docker-compose ps @@ -60,7 +60,7 @@ test: @docker-compose down @rm -rf db-data-4 || echo "no permission to delete" # test with elastic search service - @RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-5 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.ElasticSearchService docker-compose up -d db keycloak registry certificate-signer certificate-api + @RELEASE_VERSION=latest KEYCLOAK_IMPORT_DIR=java/apitest/src/test/resources KEYCLOAK_SECRET=a52c5f4a-89fd-40b9-aea2-3f711f14c889 DB_DIR=db-data-5 SEARCH_PROVIDER_NAME=dev.sunbirdrc.registry.service.ElasticSearchService EXPAND_REFERENCE=false docker-compose up -d db keycloak registry certificate-signer certificate-api @echo "Starting the test" && sh build/wait_for_port.sh 8080 @echo "Starting the test" && sh build/wait_for_port.sh 8081 @docker-compose ps diff --git a/docker-compose.yml b/docker-compose.yml index a0e777e2a..cab097db4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,7 @@ services: - redis_host=redis - redis_port=6379 - manager_type=${MANAGER_TYPE-DefinitionsManager} + - expand_reference=${EXPAND_REFERENCE-false} ports: - "8081:8081" depends_on: From 6c537e6e0324bad81f4f59c94e75c0406af5b168 Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Thu, 19 Jan 2023 02:32:49 +0530 Subject: [PATCH 06/14] fetch from elastic search in batch mode instead of one by one mode --- .../service/ElasticSearchService.java | 76 ++++++++++++++----- 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java index 9ad87d6f6..68502a0c8 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java @@ -1,8 +1,7 @@ package dev.sunbirdrc.registry.service; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -118,34 +117,51 @@ public JsonNode search(JsonNode inputQueryNode) throws IOException { private ArrayNode expandReference(JsonNode searchedNode) { ArrayNode arrayNode = (ArrayNode) searchedNode; ArrayNode nodeWithExpandedReference = JsonNodeFactory.instance.arrayNode(); + HashMap> indexOsidsMap = new HashMap<>(); for (JsonNode node : arrayNode) { ObjectNode objectNode = (ObjectNode) node; - List removableReferenceKeys = new ArrayList<>(); objectNode.fields().forEachRemaining(objectField -> { if(objectField.getValue().asText().startsWith("did:")) { - String[] splits = objectField.getValue().asText().split(":"); - String indexName = splits[1].toLowerCase(); - String osid = splits[2]; - SearchQuery searchQuery = null; - ArrayNode referenceNode = null; - try { - searchQuery = getSearchQuery(splits[1], osid); - referenceNode = (ArrayNode) elasticService.search(indexName, searchQuery); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - if(referenceNode.size() > 0) { - objectField.setValue(referenceNode.get(0)); + String[] referenceStrSplit = objectField.getValue().asText().split(":"); + String indexName = referenceStrSplit[1].toLowerCase(); + String osid = referenceStrSplit[2]; + List osids; + if(indexOsidsMap.get(indexName) == null) { + osids = new ArrayList(); } else { - removableReferenceKeys.add(objectField.getKey()); + osids = indexOsidsMap.get(indexName); } + osids.add(osid); + indexOsidsMap.put(indexName, osids); } }); - for (String referenceKeys: removableReferenceKeys) { - objectNode.remove(referenceKeys); + } + SearchQuery searchQuery = null; + ArrayNode referenceNodes = JsonNodeFactory.instance.arrayNode(); + for (Map.Entry> indexOsidEntry: indexOsidsMap.entrySet()) { + try { + searchQuery = getSearchQuery(indexOsidEntry.getKey(), indexOsidEntry.getValue()); + referenceNodes.addAll((ArrayNode) elasticService.search(indexOsidEntry.getKey(), searchQuery)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); } + } + for (JsonNode node : arrayNode) { + ObjectNode objectNode = (ObjectNode) node; + ArrayNode finalReferenceNodes = referenceNodes; + objectNode.fields().forEachRemaining(objectField -> { + if (objectField.getValue().asText().startsWith("did:")) { + String[] referenceStrSplit = objectField.getValue().asText().split(":"); + String osid = referenceStrSplit[2]; + for(JsonNode referenceNode: finalReferenceNodes) { + if(referenceNode.get("osid").textValue().equals(osid)) { + objectNode.set(objectField.getKey(), referenceNode); + } + } + } + }); nodeWithExpandedReference.add(objectNode); } return nodeWithExpandedReference; @@ -161,6 +177,24 @@ private SearchQuery getSearchQuery(String entityName, String osid) throws JsonPr return searchQuery1; } + private SearchQuery getSearchQuery(String entityName, List osids) throws JsonProcessingException { + ArrayNode osidsArrayNode = JsonNodeFactory.instance.arrayNode(); + for (String osid: osids) { + osidsArrayNode.add(osid); + } + ObjectNode objectNode = JsonNodeFactory.instance.objectNode(); + ObjectNode conditionNode = JsonNodeFactory.instance.objectNode(); + conditionNode.set("or", osidsArrayNode); + ObjectNode osidFilterNode = JsonNodeFactory.instance.objectNode(); + osidFilterNode.set("osid", conditionNode); + objectNode.set("filters", osidFilterNode); + ArrayNode entity = JsonNodeFactory.instance.arrayNode(); + entity.add(entityName); + objectNode.set(ENTITY_TYPE, entity); + SearchQuery searchQuery1 = getSearchQuery(objectNode, offset, limit); + return searchQuery1; + } + private void updateStatusFilter(SearchQuery searchQuery) { List filterList = searchQuery.getFilters(); Filter filter = new Filter(Constants.STATUS_KEYWORD, FilterOperators.neq, Constants.STATUS_INACTIVE); From 876ad9a42b4ceb4b191cc9f1efbbdaf1f0d73391 Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Fri, 20 Jan 2023 01:52:25 +0530 Subject: [PATCH 07/14] fix: using shard id in nativesearchservice --- .../main/java/dev/sunbirdrc/registry/dao/VertexWriter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java index 6103110d5..f21e39310 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java @@ -2,10 +2,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import dev.sunbirdrc.registry.middleware.util.Constants; import dev.sunbirdrc.registry.sink.DatabaseProvider; import dev.sunbirdrc.registry.util.ArrayHelper; +import dev.sunbirdrc.registry.util.RecordIdentifier; import dev.sunbirdrc.registry.util.RefLabelHelper; import dev.sunbirdrc.registry.util.TypePropertyHelper; import org.apache.tinkerpop.gremlin.process.traversal.P; @@ -237,11 +239,12 @@ private Vertex processNode(String label, JsonNode jsonObject) { private void updateParentForReferencingNode(Vertex vertex, Map.Entry entry, JsonNode entryValue) { String[] dids = entryValue.asText().split(DID_SEPERATOR); - Iterator vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, dids[2]); + String osid = RecordIdentifier.parse(dids[2]).getUuid(); + Iterator vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, osid); while(vertexIterator.hasNext()) { Vertex dependent = vertexIterator.next(); addEdge(entry.getKey(), vertex, dependent); - vertex.property(RefLabelHelper.getLabel(entry.getKey(), uuidPropertyName), dids[2]); + vertex.property(RefLabelHelper.getLabel(entry.getKey(), uuidPropertyName), osid); } } From 5c2481b1ce21a4726ef463995772613e65cb0447 Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Fri, 20 Jan 2023 17:16:58 +0530 Subject: [PATCH 08/14] change logic for expanding reference entity --- .../test/java/e2e/registry/registry.feature | 1 + .../registry/dao/RegistryDaoImpl.java | 8 ++-- .../sunbirdrc/registry/dao/VertexReader.java | 43 +++++++++++++++---- .../sunbirdrc/registry/dao/VertexWriter.java | 8 +--- .../service/impl/RegistryServiceImpl.java | 8 ++-- .../registry/util/EntityParenter.java | 6 +-- .../registry/dao/VertexWriterTest.java | 9 ++-- .../registry/dao/impl/SearchDaoImplTest.java | 2 +- .../service/impl/RegistryServiceImplTest.java | 8 ++-- 9 files changed, 57 insertions(+), 36 deletions(-) diff --git a/java/apitest/src/test/java/e2e/registry/registry.feature b/java/apitest/src/test/java/e2e/registry/registry.feature index 3bee5a64e..b2ca6c045 100644 --- a/java/apitest/src/test/java/e2e/registry/registry.feature +++ b/java/apitest/src/test/java/e2e/registry/registry.feature @@ -327,4 +327,5 @@ Feature: Registry api tests When method get Then status 200 And response[0].osid.length > 0 + And match response[0].references == '#present' diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java index 2e701cd49..93ca049ea 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/RegistryDaoImpl.java @@ -52,7 +52,7 @@ public DatabaseProvider getDatabaseProvider() { * @return */ public String addEntity(Graph graph, JsonNode rootNode) { - VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName, expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName); String entityId = vertexWriter.writeNodeEntity(rootNode); return entityId; } @@ -66,7 +66,7 @@ public String addEntity(Graph graph, JsonNode rootNode) { */ public JsonNode getEntity(Graph graph, String entityType, String uuid, ReadConfigurator readConfigurator) throws Exception { - VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager); + VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager, expandReferenceObj); JsonNode result = vr.read(entityType, uuid); return result; @@ -75,7 +75,7 @@ public JsonNode getEntity(Graph graph, String entityType, String uuid, ReadConfi public JsonNode getEntity(Graph graph, Vertex vertex, ReadConfigurator readConfigurator, boolean expandInternal) throws Exception { - VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager); + VertexReader vr = new VertexReader(getDatabaseProvider(), graph, readConfigurator, uuidPropertyName, definitionsManager, expandReferenceObj); ObjectNode constructObject = vr.constructObject(vertex); if (expandInternal) { String entityType = (String) ValueType.getValue(constructObject.get(TypePropertyHelper.getTypeName())); @@ -106,7 +106,7 @@ public void updateVertex(Graph graph, Vertex vertex, JsonNode inputJsonNode, Str if (databaseProvider.getId(vertex).equals(osidVal)) { updateObject(graph, vertex, (ObjectNode) inputJsonNode); } else { - VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName, expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, getDatabaseProvider(), uuidPropertyName); if(inputJsonNode.get(uuidPropertyName) != null) { vertexWriter.writeSingleNode(vertex, objectName, inputJsonNode.get(objectName)); } else { diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java index e1329c47c..b337fae79 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java @@ -8,12 +8,7 @@ import dev.sunbirdrc.registry.middleware.util.Constants; import dev.sunbirdrc.registry.middleware.util.JSONUtil; import dev.sunbirdrc.registry.sink.DatabaseProvider; -import dev.sunbirdrc.registry.util.ArrayHelper; -import dev.sunbirdrc.registry.util.Definition; -import dev.sunbirdrc.registry.util.IDefinitionsManager; -import dev.sunbirdrc.registry.util.ReadConfigurator; -import dev.sunbirdrc.registry.util.RefLabelHelper; -import dev.sunbirdrc.registry.util.TypePropertyHelper; +import dev.sunbirdrc.registry.util.*; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Graph; @@ -32,6 +27,8 @@ import java.util.NoSuchElementException; import java.util.Set; +import static dev.sunbirdrc.registry.middleware.util.Constants.DID_TYPE; + /** * Given a vertex from the graph, constructs a json out it */ @@ -46,15 +43,18 @@ public class VertexReader { private Vertex rootVertex; private LinkedHashMap uuidVertexMap = new LinkedHashMap<>(); + private boolean expandReferenceObj; private Logger logger = LoggerFactory.getLogger(VertexReader.class); + public VertexReader(DatabaseProvider databaseProvider, Graph graph, ReadConfigurator configurator, String uuidPropertyName, - IDefinitionsManager definitionsManager) { + IDefinitionsManager definitionsManager, boolean expandReferenceObj) { this.databaseProvider = databaseProvider; this.graph = graph; this.configurator = configurator; this.uuidPropertyName = uuidPropertyName; this.definitionsManager = definitionsManager; + this.expandReferenceObj = expandReferenceObj; } /** @@ -460,7 +460,7 @@ public JsonNode readInternal(Vertex rootVertex) throws Exception { throw new RecordNotFoundException("entity status is inactive"); } ObjectNode rootNode = constructObject(rootVertex); - entityType = (String) ValueType.getValue(rootNode.get(TypePropertyHelper.getTypeName())); + String entityType = (String) ValueType.getValue(rootNode.get(TypePropertyHelper.getTypeName())); // Set the type for the root node, so as to wrap. populateMaps(rootNode, rootVertex); @@ -485,14 +485,39 @@ public JsonNode readInternal(Vertex rootVertex) throws Exception { // objects. // The properties could exist anywhere. Refer to the local arrMap. expandChildObject(rootNode, 0); - + if(expandReferenceObj) + expandReferenceNodes(rootNode); entityNode.set(entityType, rootNode); + // After reading the entire type, now trim the @type property trimAttributes(entityNode); return entityNode; } + private void expandReferenceNodes(ObjectNode rootNode) { + rootNode.fields().forEachRemaining(entry -> { + if(entry.getValue().isValueNode() && entry.getValue().asText().startsWith(DID_TYPE)) { + String[] dids = entry.getValue().asText().split(":"); + String osid = RecordIdentifier.parse(dids[2]).getUuid(); + Iterator vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, osid); + while (vertexIterator.hasNext()) { + Vertex dependent = vertexIterator.next(); + ObjectNode references; + try { + references = (ObjectNode) readInternal(dependent); + references = (ObjectNode) references.get(references.fieldNames().next()); + } catch (Exception e) { + throw new RuntimeException(e); + } + if(references != null) { + entry.setValue(references); + } + } + } + }); + } + /** * Trims out local helper attributes like the type, uuid depending on the * ReadConfigurator diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java index f21e39310..e7529c338 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java @@ -38,13 +38,11 @@ public class VertexWriter { private static final String EMPTY_STR = ""; private static final String DID_SEPERATOR = ":"; private Logger logger = LoggerFactory.getLogger(VertexWriter.class); - private boolean expandReferenceObj; - public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName, boolean expandReferenceObj) { + public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName) { this.graph = graph; this.databaseProvider = databaseProvider; this.uuidPropertyName = uuidPropertyName; - this.expandReferenceObj = expandReferenceObj; } /** @@ -222,9 +220,7 @@ private Vertex processNode(String label, JsonNode jsonObject) { jsonObject.fields().forEachRemaining(entry -> { JsonNode entryValue = entry.getValue(); logger.debug("Processing {} -> {}", entry.getKey(), entry.getValue()); - if (entryValue.isValueNode() && entryValue.asText().startsWith(DID_TYPE) && expandReferenceObj) { - updateParentForReferencingNode(vertex, entry, entryValue); - } else if (entryValue.isValueNode()) { + if (entryValue.isValueNode()) { // Directly add under the vertex as a property vertex.property(entry.getKey(), ValueType.getValue(entryValue)); } else if (entryValue.isObject()) { diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java index 57925a933..5857716f9 100755 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImpl.java @@ -149,7 +149,7 @@ public Vertex deleteEntityById(Shard shard, String userId, String uuid) throws E Graph graph = osGraph.getGraphStore(); try (Transaction tx = databaseProvider.startTransaction(graph)) { ReadConfigurator configurator = ReadConfiguratorFactory.getOne(false); - VertexReader vertexReader = new VertexReader(databaseProvider, graph, configurator, uuidPropertyName, definitionsManager); + VertexReader vertexReader = new VertexReader(databaseProvider, graph, configurator, uuidPropertyName, definitionsManager, expandReferenceObj); Vertex vertex = vertexReader.getVertex(null, uuid); String index = vertex.property(Constants.TYPE_STR_JSON_LD).isPresent() ? (String) vertex.property(Constants.TYPE_STR_JSON_LD).value() : null; if (!StringUtils.isEmpty(index) && index.equals(Schema)) { @@ -278,7 +278,7 @@ public void updateEntity(Shard shard, String userId, String id, String jsonStrin // Read the node and // TODO - decrypt properties to pass validation ReadConfigurator readConfigurator = ReadConfiguratorFactory.getForUpdateValidation(); - VertexReader vr = new VertexReader(databaseProvider, graph, readConfigurator, uuidPropertyName, definitionsManager); + VertexReader vr = new VertexReader(databaseProvider, graph, readConfigurator, uuidPropertyName, definitionsManager, expandReferenceObj); JsonNode readNode = vr.read(entityType, id); String rootId = readNode.findPath(Constants.ROOT_KEYWORD).textValue(); @@ -401,7 +401,7 @@ private void doUpdateArray(Shard shard, Graph graph, IRegistryDao registryDao, V Set updatedUuids = new HashSet(); Set previousArrayItemsUuids = vr.getArrayItemUuids(blankArrVertex); - VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName, expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName); for (JsonNode item : arrayNode) { if (item.isObject()) { @@ -493,7 +493,7 @@ private void doUpdate(Shard shard, Graph graph, IRegistryDao registryDao, Vertex // updateArrayItems one by one doUpdateArray(shard, graph, registryDao, vr, existArrayVertex, (ArrayNode) oneElementNode, userInputKey); } else { - VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName, expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, shard.getDatabaseProvider(), uuidPropertyName); vertexWriter.createArrayNode(rootVertex, oneElement.getKey(), (ArrayNode) oneElementNode); } registryDao.updateVertex(graph, existArrayVertex, oneElementNode, oneElement.getKey()); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java index 259844bf6..72e7269db 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/util/EntityParenter.java @@ -35,8 +35,6 @@ public class EntityParenter { @Value("${database.uuidPropertyName}") public String uuidPropertyName; - @Value("${registry.expandReference}") - public boolean expandReferenceObj; @Autowired private DBProviderFactory dbProviderFactory; @@ -137,7 +135,7 @@ public Optional ensureKnownParenters() { String parentLabel = ParentLabelGenerator.getLabel(defintionName); parentLabels.add(parentLabel); - VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName, expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName); Vertex v = vertexWriter.ensureParentVertex(parentLabel); ShardParentInfo shardParentInfo = new ShardParentInfo(defintionName, v); @@ -294,7 +292,7 @@ private void updateParentVertexIndexProperties(DatabaseProvider dbProvider, Vert Graph graph = osGraph.getGraphStore(); try (Transaction tx = dbProvider.startTransaction(graph)) { - VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName, expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, dbProvider, uuidPropertyName); Vertex v = graph.vertices(parentVertex.id()).next(); vertexWriter.updateParentIndexProperty(v, Constants.INDEX_FIELDS, indexFields); diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java index 89b2c324c..c7d765ae5 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/VertexWriterTest.java @@ -56,8 +56,6 @@ public class VertexWriterTest { private VertexWriter vertexWriter; Vertex vertex; - @Value("${registry.expandReference}") - private boolean expandReferenceObj; @Before public void setUp() throws Exception { @@ -69,7 +67,7 @@ public void setUp() throws Exception { Mockito.when(osGraph.getGraphStore()).thenReturn(graph); vertex = Mockito.mock(Vertex.class); Mockito.when(graph.addVertex(anyString())).thenReturn(vertex); - vertexWriter = new VertexWriter(graph, mockDatabaseProvider, testUuidPropertyName, expandReferenceObj); + vertexWriter = new VertexWriter(graph, mockDatabaseProvider, testUuidPropertyName); } @Test @@ -124,9 +122,12 @@ public void writeSingleNode() { ObjectNode entryValue = JsonNodeFactory.instance.objectNode(); TextNode value = JsonNodeFactory.instance.textNode("value1"); entryValue.set("field1", value); + TextNode references = JsonNodeFactory.instance.textNode("did:some_entity:123"); + entryValue.set("references", references); Mockito.when(mockDatabaseProvider.getId(vertex)).thenReturn("123"); Vertex actualVertex = vertexWriter.writeSingleNode(parentVertex, label, entryValue); - + Mockito.verify(vertex, Mockito.times(1)).property("references", "did:some_entity:123"); + Mockito.verify(vertex, Mockito.times(1)).property("field1", "value1"); Mockito.verify(parentVertex, Mockito.times(1)).addEdge(label, vertex); Mockito.verify(parentVertex, Mockito.times(1)).property(label + "_" + testUuidPropertyName, "123"); assertEquals(vertex, actualVertex); diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java index 30f4d2900..e2cefb05f 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/dao/impl/SearchDaoImplTest.java @@ -180,7 +180,7 @@ private SearchQuery getSearchQuery(List rootTypes, String property, Obje } private void populateGraph() { - VertexWriter vertexWriter = new VertexWriter(graph, databaseProvider, "tid", expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, databaseProvider, "tid"); Vertex v1 = vertexWriter.createVertex("Teacher"); v1.property("serialNum", 1); v1.property("teacherName", "marko"); diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java index 3c470988d..fea4d22df 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java @@ -164,7 +164,7 @@ public void initialize() throws IOException { } private void populateGraph() { - VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid", expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid"); Vertex v1 = vertexWriter.createVertex("Teacher"); v1.property("serialNum", 1); v1.property("teacherName", "marko"); @@ -372,20 +372,20 @@ public void shouldNotRemoveAnyFieldsInUpdate() throws Exception { public void shouldTestVertexWriter() throws Exception { String v1 = addStudentToGraph(); ReadConfigurator readConfigurator = ReadConfiguratorFactory.getForUpdateValidation(); - VertexReader vertexReader = new VertexReader(mockDatabaseProvider, graph, readConfigurator, "osid", definitionsManager); + VertexReader vertexReader = new VertexReader(mockDatabaseProvider, graph, readConfigurator, "osid", definitionsManager, expandReferenceObj); JsonNode student = vertexReader.read("Student", v1); assertNotNull(student); } private String addStudentToGraph() throws JsonProcessingException { - VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid", expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid"); return vertexWriter.writeNodeEntity(objectMapper.readTree("{\"Student\": {\n" + " \"name\": \"abc\"\n" + "}}")); } private String addTeacherToGraph() throws JsonProcessingException { - VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid", expandReferenceObj); + VertexWriter vertexWriter = new VertexWriter(graph, mockDatabaseProvider, "osid"); return vertexWriter.writeNodeEntity(objectMapper.readTree("{\"Teacher\": {\n" + " \"fullName\": \"abc\"\n" + "}}")); From 63cbd6723c90948dee96e8c2f07d019fe1b186cf Mon Sep 17 00:00:00 2001 From: Tejas Varade Date: Fri, 20 Jan 2023 17:21:29 +0530 Subject: [PATCH 09/14] refactor: remove unwanted method --- .../dev/sunbirdrc/registry/dao/VertexWriter.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java index e7529c338..f34939e22 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexWriter.java @@ -36,7 +36,6 @@ public class VertexWriter { private DatabaseProvider databaseProvider; private String parentOSid; private static final String EMPTY_STR = ""; - private static final String DID_SEPERATOR = ":"; private Logger logger = LoggerFactory.getLogger(VertexWriter.class); public VertexWriter(Graph graph, DatabaseProvider databaseProvider, String uuidPropertyName) { @@ -233,17 +232,6 @@ private Vertex processNode(String label, JsonNode jsonObject) { return vertex; } - private void updateParentForReferencingNode(Vertex vertex, Map.Entry entry, JsonNode entryValue) { - String[] dids = entryValue.asText().split(DID_SEPERATOR); - String osid = RecordIdentifier.parse(dids[2]).getUuid(); - Iterator vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, osid); - while(vertexIterator.hasNext()) { - Vertex dependent = vertexIterator.next(); - addEdge(entry.getKey(), vertex, dependent); - vertex.property(RefLabelHelper.getLabel(entry.getKey(), uuidPropertyName), osid); - } - } - /** * Adds an edge between two vertices * From 4485020519bf659f1d9dfffb7cab8c6b0d4d49fb Mon Sep 17 00:00:00 2001 From: Sreejit-K Date: Tue, 23 Jan 2024 15:28:28 +0530 Subject: [PATCH 10/14] sample schemas for testing --- .../resources/public/_schemas/Common.json | 393 ++++++++++++++++++ .../resources/public/_schemas/Student.json | 94 +++++ .../resources/public/_schemas/Teacher.json | 101 +++++ 3 files changed, 588 insertions(+) create mode 100644 java/registry/src/main/resources/public/_schemas/Common.json create mode 100644 java/registry/src/main/resources/public/_schemas/Student.json create mode 100644 java/registry/src/main/resources/public/_schemas/Teacher.json diff --git a/java/registry/src/main/resources/public/_schemas/Common.json b/java/registry/src/main/resources/public/_schemas/Common.json new file mode 100644 index 000000000..594f49b45 --- /dev/null +++ b/java/registry/src/main/resources/public/_schemas/Common.json @@ -0,0 +1,393 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Common", + "definitions": { + "IdentityDetails": { + "type": "object", + "title": "Identity Details", + "description": "Identity Details", + "required": [], + "properties": { + "fullName": { + "$id": "#/properties/fullName", + "type": "string", + "title": "Full name" + }, + "gender": { + "$id": "#/properties/gender", + "type": "string", + "enum": [ + "Male", + "Female", + "Other" + ], + "title": "Gender" + }, + "dob": { + "$id": "#/properties/dob", + "type": "string", + "format": "date", + "title": "DOB" + }, + "identityHolder": { + "type": "object", + "properties": { + "type": { + "$id": "#/properties/type", + "type": "string", + "$comment": "Nationality", + "title": "ID Type", + "enum": [ + "AADHAR", + "PAN", + "LICENSE", + "OTHER" + ] + }, + "value": { + "$id": "#/properties/value", + "type": "string", + "$comment": "Nationality", + "title": "ID Value" + } + } + } + } + }, + "ContactDetails": { + "type": "object", + "title": "Contact Details", + "description": "Contact Details", + "required": [], + "properties": { + "email": { + "$id": "#/properties/email", + "type": "string", + "title": "Email" + }, + "mobile": { + "$id": "#/properties/mobile", + "type": "string", + "title": "Mobile" + }, + "address": { + "$id": "#/properties/address", + "$ref": "#/definitions/Address", + "title": "Address" + } + } + }, + "AcademicQualification": { + "type": "object", + "properties": { + "instituteOSID": { + "$id": "#/properties/instituteOSID", + "type": "string", + "title": "Institute OSID" + }, + "instituteName": { + "$id": "#/properties/instituteName", + "type": "string", + "title": "Institute Name" + }, + "qualification": { + "$id": "#/properties/qualification", + "type": "string", + "title": "Qualification" + }, + "program": { + "$id": "#/properties/program", + "type": "string", + "title": "Program of Study" + }, + "graduationYear": { + "$id": "#/properties/graduationYear", + "type": "string", + "title": "Year of Graduation" + }, + "marks": { + "$id": "#/properties/marks", + "type": "string", + "title": "Marks / Ranking / GPA, etc" + } + } + }, + "ExperienceType": { + "type": "object", + "properties": { + "instituteOSID": { + "$id": "#/properties/instituteOSID", + "type": "string", + "title": "Institute OSID" + }, + "instituteName": { + "$id": "#/properties/instituteName", + "type": "string", + "title": "Institute Name" + }, + "employmentType": { + "$id": "#/properties/employmentType", + "type": "string", + "title": "Employment Type", + "enum": [ + "Permanent", + "Contract" + ] + }, + "start": { + "$id": "#/properties/start", + "type": "string", + "format": "date", + "title": "Start Date" + }, + "end": { + "$id": "#/properties/end", + "type": "string", + "format": "date", + "title": "End Date" + }, + "teacherType": { + "$id": "#/properties/teacherType", + "type": "string", + "title": "Teacher Type", + "enum": [ + "Head teacher", + "Acting head teacher", + "Teacher", + "Instructor positioned as per RTE", + "Principal", + "VicePrincipal", + "Lecturer" + ] + }, + "subjectsTaught": { + "$id": "#/properties/subjectsTaught", + "title": "Subjects Taught", + "type": "array", + "items": { + "type": "object", + "properties": { + "subjects": { + "$id": "#/properties/subjects", + "type": "string", + "title": "Subject", + "enum": [ + "English", + "Kannada", + "Mathematics", + "Science", + "Social Science", + "Art" + ] + }, + "grades": { + "$id": "#/properties/grades", + "type": "string", + "title": "Grades", + "enum": [ + "Pre-Pri.", + "Class I", + "Class II", + "Class III", + "Class IV", + "Class V", + "Class VI", + "Class VII", + "Class VIII", + "Class IX", + "Class X", + "Class XI", + "Class XII" + ] + } + } + } + } + } + }, + "EducationType": { + "type": "object", + "properties": { + "institute": { + "$id": "#/properties/institute", + "type": "string", + "title": "Institute Name" + }, + "instituteOSID": { + "$id": "#/properties/instituteOSID", + "type": "string", + "title": "Institute Osid" + }, + "board": { + "$id": "#/properties/board", + "type": "string", + "title": "Board of Education" + }, + "medium": { + "$id": "#/properties/program", + "type": "string", + "enum": [ + "English", + "Hindi", + "Gujarati", + "Malayalam", + "Tamil" + ], + "title": "Medium of Education" + }, + "class": { + "$id": "#/properties/graduationYear", + "type": "string", + "enum": [ + "Pre-Pri.", + "Class I", + "Class II", + "Class III", + "Class IV", + "Class V", + "Class VI", + "Class VII", + "Class VIII", + "Class IX", + "Class X", + "Class XI", + "Class XII" + ], + "title": "Current Enrollment Class / Std." + }, + "documents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fileName": { + "type": "string", + "format": "file" + } + } + }, + "title": "Education related documents" + } + } + }, + "GuardianDetails": { + "type": "object", + "properties": { + "fullName": { + "$id": "#/properties/fullName", + "type": "string", + "title": "Guardian's Full Name" + }, + "relation": { + "$id": "#/properties/relation", + "type": "string", + "title": "Guardian Relation type" + } + } + }, + "CertificationType": { + "type": "object", + "properties": { + "institute": { + "type": "string" + }, + "start": { + "type": "string", + "format": "date" + }, + "end": { + "type": "string", + "format": "date" + } + } + }, + "Document": { + "type": "object", + "properties": { + "fileName": { + "type": "string", + "format": "file" + } + } + }, + "Address": { + "type": "object", + "required": [ + ], + "additionalProperties": false, + "properties": { + "plot": { + "type": "string", + "title": "Plot" + }, + "street": { + "type": "string", + "title": "Street" + }, + "landmark": { + "type": "string", + "title": "Landmark" + }, + "locality": { + "type": "string", + "title": "Locality" + }, + "state": { + "type": "string", + "title": "State", + "enum": [ + "Andaman and Nicobar Islands", + "Andhra Pradesh", + "Arunachal Pradesh", + "Assam", + "Bihar", + "Chandigarh", + "Chhattisgarh", + "Dadra and Nagar Haveli", + "Daman and Diu", + "Delhi", + "Goa", + "Gujarat", + "Haryana", + "Himachal Pradesh", + "Jammu and Kashmir", + "Jharkhand", + "Karnataka", + "Kerala", + "Ladakh", + "Lakshadweep", + "Madhya Pradesh", + "Maharashtra", + "Manipur", + "Meghalaya", + "Mizoram", + "Nagaland", + "Odisha", + "Puducherry", + "Punjab", + "Rajasthan", + "Sikkim", + "Tamil Nadu", + "Telangana", + "Tripura", + "Uttar Pradesh", + "Uttarakhand", + "West Bengal" + ] + }, + "district": { + "type": "string", + "title": "District" + }, + "village": { + "type": "string", + "title": "Village/Town/City" + }, + "pincode": { + "type": "string", + "title": "Pincode" + } + } + } + } + } \ No newline at end of file diff --git a/java/registry/src/main/resources/public/_schemas/Student.json b/java/registry/src/main/resources/public/_schemas/Student.json new file mode 100644 index 000000000..c974898ef --- /dev/null +++ b/java/registry/src/main/resources/public/_schemas/Student.json @@ -0,0 +1,94 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "Student": { + "$ref": "#/definitions/Student" + } + }, + "required": [ + "Student" + ], + "title":"Student", + "definitions": { + "Student": { + "$id": "#/properties/Student", + "type": "object", + "title": "The Student Schema", + "required": [ + + ], + "properties": { + "identityDetails": { + "$ref": "Common.json/#/definitions/IdentityDetails" + }, + "contactDetails": { + "$ref": "Common.json/#/definitions/ContactDetails" + }, + "educationDetails": { + "type": "array", + "items": { + "$ref": "Common.json/#/definitions/EducationType" + }, + "title": "Education Details" + }, + "guardianDetails": { + "$ref": "Common.json/#/definitions/GuardianDetails", + "title": "Guardian Details" + } + } + } + }, + "_osConfig": { + "osComment": ["This section contains the OpenSABER specific configuration information", + "privateFields: Optional; list of field names to be encrypted and stored in database", + "signedFields: Optional; list of field names that must be pre-signed", + "indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.", + "uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name", + "systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid"], + + "privateFields": [ + "$.identityDetails.dob", + "$.identityDetails.identityType", + "$.identityDetails.identityValue", + "$.contactDetails.email", + "$.contactDetails.mobile", + "$.contactDetails.address.plot", + "$.contactDetails.address.street", + "$.contactDetails.address.landmark", + "$.contactDetails.address.locality" + ], + "indexFields": ["studentName"], + "uniqueIndexFields": ["identityValue"], + "systemFields": [ + "osCreatedAt", + "osUpdatedAt", + "osCreatedBy", + "osUpdatedBy" + ], + "attestationPolicies": [ + { + "name": "studentInstituteAttest", + "additionalInput": { + "enrollmentNumber": {"type": "string"} + }, + "attestationProperties": { + "name": "$.identityDetails.fullName", + "educationDetails": "$.educationDetails" + }, + "type": "MANUAL", + "conditions": "(ATTESTOR#$.experience.[*].instituteOSID#.contains(REQUESTER#$.educationDetails.*.instituteOSID#))", + "attestorPlugin": "did:internal:ClaimPluginActor?entity=Teacher" + } + ], + "inviteRoles": ["anonymous"], + "roles" : ["anonymous"], + "ownershipAttributes": [ + { + "email": "/contactDetails/email", + "mobile": "/contactDetails/mobile", + "userId": "/contactDetails/mobile" + } + ] + } + } \ No newline at end of file diff --git a/java/registry/src/main/resources/public/_schemas/Teacher.json b/java/registry/src/main/resources/public/_schemas/Teacher.json new file mode 100644 index 000000000..ceca50ef2 --- /dev/null +++ b/java/registry/src/main/resources/public/_schemas/Teacher.json @@ -0,0 +1,101 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "Teacher": { + "$ref": "#/definitions/Teacher" + } + }, + "required": [ + "Teacher" + ], + "title": "Teacher", + "definitions": { + "Teacher": { + "$id": "#/properties/Teacher", + "type": "object", + "title": "The Teacher Schema", + "required": [ + + ], + "properties": { + "identityDetails": { + "$ref": "Common.json/#/definitions/IdentityDetails" + }, + "contactDetails": { + "$ref": "Common.json/#/definitions/ContactDetails" + }, + "academicQualifications": { + "type": "array", + "items": { + "$ref": "Common.json/#/definitions/AcademicQualification" + }, + "title": "Education Details" + }, + "experience": { + "type": "array", + "items": { + "$ref": "Common.json/#/definitions/ExperienceType" + }, + "title": "Professional Experience" + } + } + } + }, + "_osConfig": { + "osComment": [ + "This section contains the OpenSABER specific configuration information", + "privateFields: Optional; list of field names to be encrypted and stored in database", + "signedFields: Optional; list of field names that must be pre-signed", + "indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, teacherCode) is a composite index and teacherName is a single column index.", + "uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name", + "systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid" + ], + "privateFields": [ + "$.identityDetails.dob", + "$.identityDetails.identityHolder.value", + "$.contactDetails.email", + "$.contactDetails.mobile", + "$.contactDetails.address.plot", + "$.contactDetails.address.street", + "$.contactDetails.address.landmark", + "$.contactDetails.address.locality" + ], + "internalFields": ["$.identityDetails.identityHolder.type"], + "indexFields": [], + "uniqueIndexFields": [], + "systemFields": [ + "osCreatedAt", + "osUpdatedAt", + "osCreatedBy", + "osUpdatedBy" + ], + "attestationPolicies": [ + { + "name": "teacherExperience", + "conditions": "(ATTESTOR#$.osid#.contains(REQUESTER#$.experience.*.instituteOSID#))", + "type": "MANUAL", + "attestorPlugin": "did:internal:ClaimPluginActor?entity=Institute", + "attestationProperties": { + "experience": "$.experience" + } + }, + { + "name": "teacherAcademicsQualification", + "conditions": "(ATTESTOR#$.osid#.contains(REQUESTER#$.academicQualifications.*.instituteOSID#))", + "type": "MANUAL", + "attestorPlugin": "did:internal:ClaimPluginActor?entity=Institute", + "attestationProperties": { + "academicQualifications": "$.academicQualifications" + } + } + ], + "ownershipAttributes": [ + { + "email": "/contactDetails/email", + "mobile": "/contactDetails/mobile", + "userId": "/contactDetails/mobile" + } + ] + } + } \ No newline at end of file From 95bd6f5f89bf771a6d0a14582f712b186be937f6 Mon Sep 17 00:00:00 2001 From: Sreejit-K Date: Sun, 28 Jan 2024 17:50:11 +0530 Subject: [PATCH 11/14] Added changes to removie private feilds in the reference node --- .../src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java | 1 + .../dev/sunbirdrc/registry/service/ElasticSearchService.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java index ed1a3b257..6a204f87f 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java @@ -507,6 +507,7 @@ private void expandReferenceNodes(ObjectNode rootNode) { try { references = (ObjectNode) readInternal(dependent); references = (ObjectNode) references.get(references.fieldNames().next()); + references = (ObjectNode) JSONUtil.removeNodesByPath(references, definitionsManager.getExcludingFieldsForEntity(dids[1])); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java index bca33f5e4..5afa0ed43 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java @@ -161,7 +161,7 @@ private ArrayNode expandReference(JsonNode searchedNode) { String[] referenceStrSplit = objectField.getValue().asText().split(":"); String osid = referenceStrSplit[2]; for(JsonNode referenceNode: finalReferenceNodes) { - if(referenceNode.get("osid").textValue().equals(osid)) { + if(referenceNode.get("osid").textValue().contains(osid)) { objectNode.set(objectField.getKey(), referenceNode); } } From 5454b73711036f2224083b5c5ab9000d1ed75e7d Mon Sep 17 00:00:00 2001 From: Sreejit-K Date: Mon, 29 Jan 2024 09:37:58 +0530 Subject: [PATCH 12/14] Fixed test cases --- .../java/dev/sunbirdrc/registry/helper/RegistryHelper.java | 2 +- .../registry/service/impl/RegistryServiceImplTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java b/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java index a5ab367ef..bbb924232 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/helper/RegistryHelper.java @@ -1057,7 +1057,7 @@ public Vertex deleteEntity(String entityName, String entityId, String userId) th Shard shard = shardManager.activateShard(shardId); ReadConfigurator configurator = ReadConfiguratorFactory.getOne(false); Vertex vertex = registryService.deleteEntityById(shard, entityName, userId, recordId.getUuid()); - VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager); + VertexReader vertexReader = new VertexReader(shard.getDatabaseProvider(), vertex.graph(), configurator, uuidPropertyName, definitionsManager, true); JsonNode deletedNode = JsonNodeFactory.instance.objectNode().set(entityName, vertexReader.constructObject(vertex)); if(notificationEnabled) notificationHelper.sendNotification(deletedNode, DELETE); return vertex; diff --git a/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java b/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java index 3f25458e5..2afc98c5e 100644 --- a/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java +++ b/java/registry/src/test/java/dev/sunbirdrc/registry/service/impl/RegistryServiceImplTest.java @@ -334,7 +334,7 @@ public void shouldUpdateArrayFieldsInEntity() throws Exception { when(shard.getDatabaseProvider()).thenReturn(mockDatabaseProvider); String instituteOsid = addInstituteToGraph(); ReadConfigurator readConfigurator = ReadConfiguratorFactory.getForUpdateValidation(); - VertexReader vertexReader = new VertexReader(mockDatabaseProvider, graph, readConfigurator, "osid", definitionsManager); + VertexReader vertexReader = new VertexReader(mockDatabaseProvider, graph, readConfigurator, "osid", definitionsManager, true ); JsonNode instituteNode = vertexReader.read("Institute", instituteOsid); ObjectNode affiliationNode = (ObjectNode) instituteNode.get("Institute").get("affiliation").get(0); ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode(); @@ -365,7 +365,7 @@ public void shouldUpdateTextFieldsInEntity() throws Exception { when(shard.getDatabaseProvider()).thenReturn(mockDatabaseProvider); String instituteOsid = addInstituteToGraph(); ReadConfigurator readConfigurator = ReadConfiguratorFactory.getForUpdateValidation(); - VertexReader vertexReader = new VertexReader(mockDatabaseProvider, graph, readConfigurator, "osid", definitionsManager); + VertexReader vertexReader = new VertexReader(mockDatabaseProvider, graph, readConfigurator, "osid", definitionsManager, true); JsonNode instituteNode = vertexReader.read("Institute", instituteOsid); ((ObjectNode)instituteNode.get("Institute")).set("instituteName", JsonNodeFactory.instance.textNode("Holy Cross")); when(shard.getShardLabel()).thenReturn(""); From 73012801c45f379afc2b67e8072370d41e5dac42 Mon Sep 17 00:00:00 2001 From: Sreejit-K Date: Mon, 29 Jan 2024 09:44:40 +0530 Subject: [PATCH 13/14] Removed schemas that were added for testing --- .../resources/public/_schemas/Common.json | 393 ------------------ .../resources/public/_schemas/Student.json | 94 ----- .../resources/public/_schemas/Teacher.json | 101 ----- 3 files changed, 588 deletions(-) delete mode 100644 java/registry/src/main/resources/public/_schemas/Common.json delete mode 100644 java/registry/src/main/resources/public/_schemas/Student.json delete mode 100644 java/registry/src/main/resources/public/_schemas/Teacher.json diff --git a/java/registry/src/main/resources/public/_schemas/Common.json b/java/registry/src/main/resources/public/_schemas/Common.json deleted file mode 100644 index 594f49b45..000000000 --- a/java/registry/src/main/resources/public/_schemas/Common.json +++ /dev/null @@ -1,393 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "Common", - "definitions": { - "IdentityDetails": { - "type": "object", - "title": "Identity Details", - "description": "Identity Details", - "required": [], - "properties": { - "fullName": { - "$id": "#/properties/fullName", - "type": "string", - "title": "Full name" - }, - "gender": { - "$id": "#/properties/gender", - "type": "string", - "enum": [ - "Male", - "Female", - "Other" - ], - "title": "Gender" - }, - "dob": { - "$id": "#/properties/dob", - "type": "string", - "format": "date", - "title": "DOB" - }, - "identityHolder": { - "type": "object", - "properties": { - "type": { - "$id": "#/properties/type", - "type": "string", - "$comment": "Nationality", - "title": "ID Type", - "enum": [ - "AADHAR", - "PAN", - "LICENSE", - "OTHER" - ] - }, - "value": { - "$id": "#/properties/value", - "type": "string", - "$comment": "Nationality", - "title": "ID Value" - } - } - } - } - }, - "ContactDetails": { - "type": "object", - "title": "Contact Details", - "description": "Contact Details", - "required": [], - "properties": { - "email": { - "$id": "#/properties/email", - "type": "string", - "title": "Email" - }, - "mobile": { - "$id": "#/properties/mobile", - "type": "string", - "title": "Mobile" - }, - "address": { - "$id": "#/properties/address", - "$ref": "#/definitions/Address", - "title": "Address" - } - } - }, - "AcademicQualification": { - "type": "object", - "properties": { - "instituteOSID": { - "$id": "#/properties/instituteOSID", - "type": "string", - "title": "Institute OSID" - }, - "instituteName": { - "$id": "#/properties/instituteName", - "type": "string", - "title": "Institute Name" - }, - "qualification": { - "$id": "#/properties/qualification", - "type": "string", - "title": "Qualification" - }, - "program": { - "$id": "#/properties/program", - "type": "string", - "title": "Program of Study" - }, - "graduationYear": { - "$id": "#/properties/graduationYear", - "type": "string", - "title": "Year of Graduation" - }, - "marks": { - "$id": "#/properties/marks", - "type": "string", - "title": "Marks / Ranking / GPA, etc" - } - } - }, - "ExperienceType": { - "type": "object", - "properties": { - "instituteOSID": { - "$id": "#/properties/instituteOSID", - "type": "string", - "title": "Institute OSID" - }, - "instituteName": { - "$id": "#/properties/instituteName", - "type": "string", - "title": "Institute Name" - }, - "employmentType": { - "$id": "#/properties/employmentType", - "type": "string", - "title": "Employment Type", - "enum": [ - "Permanent", - "Contract" - ] - }, - "start": { - "$id": "#/properties/start", - "type": "string", - "format": "date", - "title": "Start Date" - }, - "end": { - "$id": "#/properties/end", - "type": "string", - "format": "date", - "title": "End Date" - }, - "teacherType": { - "$id": "#/properties/teacherType", - "type": "string", - "title": "Teacher Type", - "enum": [ - "Head teacher", - "Acting head teacher", - "Teacher", - "Instructor positioned as per RTE", - "Principal", - "VicePrincipal", - "Lecturer" - ] - }, - "subjectsTaught": { - "$id": "#/properties/subjectsTaught", - "title": "Subjects Taught", - "type": "array", - "items": { - "type": "object", - "properties": { - "subjects": { - "$id": "#/properties/subjects", - "type": "string", - "title": "Subject", - "enum": [ - "English", - "Kannada", - "Mathematics", - "Science", - "Social Science", - "Art" - ] - }, - "grades": { - "$id": "#/properties/grades", - "type": "string", - "title": "Grades", - "enum": [ - "Pre-Pri.", - "Class I", - "Class II", - "Class III", - "Class IV", - "Class V", - "Class VI", - "Class VII", - "Class VIII", - "Class IX", - "Class X", - "Class XI", - "Class XII" - ] - } - } - } - } - } - }, - "EducationType": { - "type": "object", - "properties": { - "institute": { - "$id": "#/properties/institute", - "type": "string", - "title": "Institute Name" - }, - "instituteOSID": { - "$id": "#/properties/instituteOSID", - "type": "string", - "title": "Institute Osid" - }, - "board": { - "$id": "#/properties/board", - "type": "string", - "title": "Board of Education" - }, - "medium": { - "$id": "#/properties/program", - "type": "string", - "enum": [ - "English", - "Hindi", - "Gujarati", - "Malayalam", - "Tamil" - ], - "title": "Medium of Education" - }, - "class": { - "$id": "#/properties/graduationYear", - "type": "string", - "enum": [ - "Pre-Pri.", - "Class I", - "Class II", - "Class III", - "Class IV", - "Class V", - "Class VI", - "Class VII", - "Class VIII", - "Class IX", - "Class X", - "Class XI", - "Class XII" - ], - "title": "Current Enrollment Class / Std." - }, - "documents": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fileName": { - "type": "string", - "format": "file" - } - } - }, - "title": "Education related documents" - } - } - }, - "GuardianDetails": { - "type": "object", - "properties": { - "fullName": { - "$id": "#/properties/fullName", - "type": "string", - "title": "Guardian's Full Name" - }, - "relation": { - "$id": "#/properties/relation", - "type": "string", - "title": "Guardian Relation type" - } - } - }, - "CertificationType": { - "type": "object", - "properties": { - "institute": { - "type": "string" - }, - "start": { - "type": "string", - "format": "date" - }, - "end": { - "type": "string", - "format": "date" - } - } - }, - "Document": { - "type": "object", - "properties": { - "fileName": { - "type": "string", - "format": "file" - } - } - }, - "Address": { - "type": "object", - "required": [ - ], - "additionalProperties": false, - "properties": { - "plot": { - "type": "string", - "title": "Plot" - }, - "street": { - "type": "string", - "title": "Street" - }, - "landmark": { - "type": "string", - "title": "Landmark" - }, - "locality": { - "type": "string", - "title": "Locality" - }, - "state": { - "type": "string", - "title": "State", - "enum": [ - "Andaman and Nicobar Islands", - "Andhra Pradesh", - "Arunachal Pradesh", - "Assam", - "Bihar", - "Chandigarh", - "Chhattisgarh", - "Dadra and Nagar Haveli", - "Daman and Diu", - "Delhi", - "Goa", - "Gujarat", - "Haryana", - "Himachal Pradesh", - "Jammu and Kashmir", - "Jharkhand", - "Karnataka", - "Kerala", - "Ladakh", - "Lakshadweep", - "Madhya Pradesh", - "Maharashtra", - "Manipur", - "Meghalaya", - "Mizoram", - "Nagaland", - "Odisha", - "Puducherry", - "Punjab", - "Rajasthan", - "Sikkim", - "Tamil Nadu", - "Telangana", - "Tripura", - "Uttar Pradesh", - "Uttarakhand", - "West Bengal" - ] - }, - "district": { - "type": "string", - "title": "District" - }, - "village": { - "type": "string", - "title": "Village/Town/City" - }, - "pincode": { - "type": "string", - "title": "Pincode" - } - } - } - } - } \ No newline at end of file diff --git a/java/registry/src/main/resources/public/_schemas/Student.json b/java/registry/src/main/resources/public/_schemas/Student.json deleted file mode 100644 index c974898ef..000000000 --- a/java/registry/src/main/resources/public/_schemas/Student.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "properties": { - "Student": { - "$ref": "#/definitions/Student" - } - }, - "required": [ - "Student" - ], - "title":"Student", - "definitions": { - "Student": { - "$id": "#/properties/Student", - "type": "object", - "title": "The Student Schema", - "required": [ - - ], - "properties": { - "identityDetails": { - "$ref": "Common.json/#/definitions/IdentityDetails" - }, - "contactDetails": { - "$ref": "Common.json/#/definitions/ContactDetails" - }, - "educationDetails": { - "type": "array", - "items": { - "$ref": "Common.json/#/definitions/EducationType" - }, - "title": "Education Details" - }, - "guardianDetails": { - "$ref": "Common.json/#/definitions/GuardianDetails", - "title": "Guardian Details" - } - } - } - }, - "_osConfig": { - "osComment": ["This section contains the OpenSABER specific configuration information", - "privateFields: Optional; list of field names to be encrypted and stored in database", - "signedFields: Optional; list of field names that must be pre-signed", - "indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, studentCode) is a composite index and studentName is a single column index.", - "uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name", - "systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid"], - - "privateFields": [ - "$.identityDetails.dob", - "$.identityDetails.identityType", - "$.identityDetails.identityValue", - "$.contactDetails.email", - "$.contactDetails.mobile", - "$.contactDetails.address.plot", - "$.contactDetails.address.street", - "$.contactDetails.address.landmark", - "$.contactDetails.address.locality" - ], - "indexFields": ["studentName"], - "uniqueIndexFields": ["identityValue"], - "systemFields": [ - "osCreatedAt", - "osUpdatedAt", - "osCreatedBy", - "osUpdatedBy" - ], - "attestationPolicies": [ - { - "name": "studentInstituteAttest", - "additionalInput": { - "enrollmentNumber": {"type": "string"} - }, - "attestationProperties": { - "name": "$.identityDetails.fullName", - "educationDetails": "$.educationDetails" - }, - "type": "MANUAL", - "conditions": "(ATTESTOR#$.experience.[*].instituteOSID#.contains(REQUESTER#$.educationDetails.*.instituteOSID#))", - "attestorPlugin": "did:internal:ClaimPluginActor?entity=Teacher" - } - ], - "inviteRoles": ["anonymous"], - "roles" : ["anonymous"], - "ownershipAttributes": [ - { - "email": "/contactDetails/email", - "mobile": "/contactDetails/mobile", - "userId": "/contactDetails/mobile" - } - ] - } - } \ No newline at end of file diff --git a/java/registry/src/main/resources/public/_schemas/Teacher.json b/java/registry/src/main/resources/public/_schemas/Teacher.json deleted file mode 100644 index ceca50ef2..000000000 --- a/java/registry/src/main/resources/public/_schemas/Teacher.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "properties": { - "Teacher": { - "$ref": "#/definitions/Teacher" - } - }, - "required": [ - "Teacher" - ], - "title": "Teacher", - "definitions": { - "Teacher": { - "$id": "#/properties/Teacher", - "type": "object", - "title": "The Teacher Schema", - "required": [ - - ], - "properties": { - "identityDetails": { - "$ref": "Common.json/#/definitions/IdentityDetails" - }, - "contactDetails": { - "$ref": "Common.json/#/definitions/ContactDetails" - }, - "academicQualifications": { - "type": "array", - "items": { - "$ref": "Common.json/#/definitions/AcademicQualification" - }, - "title": "Education Details" - }, - "experience": { - "type": "array", - "items": { - "$ref": "Common.json/#/definitions/ExperienceType" - }, - "title": "Professional Experience" - } - } - } - }, - "_osConfig": { - "osComment": [ - "This section contains the OpenSABER specific configuration information", - "privateFields: Optional; list of field names to be encrypted and stored in database", - "signedFields: Optional; list of field names that must be pre-signed", - "indexFields: Optional; list of field names used for creating index. Enclose within braces to indicate it is a composite index. In this definition, (serialNum, teacherCode) is a composite index and teacherName is a single column index.", - "uniqueIndexFields: Optional; list of field names used for creating unique index. Field names must be different from index field name", - "systemFields: Optional; list of fields names used for system standard information like created, updated timestamps and userid" - ], - "privateFields": [ - "$.identityDetails.dob", - "$.identityDetails.identityHolder.value", - "$.contactDetails.email", - "$.contactDetails.mobile", - "$.contactDetails.address.plot", - "$.contactDetails.address.street", - "$.contactDetails.address.landmark", - "$.contactDetails.address.locality" - ], - "internalFields": ["$.identityDetails.identityHolder.type"], - "indexFields": [], - "uniqueIndexFields": [], - "systemFields": [ - "osCreatedAt", - "osUpdatedAt", - "osCreatedBy", - "osUpdatedBy" - ], - "attestationPolicies": [ - { - "name": "teacherExperience", - "conditions": "(ATTESTOR#$.osid#.contains(REQUESTER#$.experience.*.instituteOSID#))", - "type": "MANUAL", - "attestorPlugin": "did:internal:ClaimPluginActor?entity=Institute", - "attestationProperties": { - "experience": "$.experience" - } - }, - { - "name": "teacherAcademicsQualification", - "conditions": "(ATTESTOR#$.osid#.contains(REQUESTER#$.academicQualifications.*.instituteOSID#))", - "type": "MANUAL", - "attestorPlugin": "did:internal:ClaimPluginActor?entity=Institute", - "attestationProperties": { - "academicQualifications": "$.academicQualifications" - } - } - ], - "ownershipAttributes": [ - { - "email": "/contactDetails/email", - "mobile": "/contactDetails/mobile", - "userId": "/contactDetails/mobile" - } - ] - } - } \ No newline at end of file From ed33b271b981772e941555daf5b57b211f9df3e1 Mon Sep 17 00:00:00 2001 From: Sreejit-K Date: Mon, 29 Jan 2024 15:04:12 +0530 Subject: [PATCH 14/14] Added regex check for reference entity --- .../main/java/dev/sunbirdrc/registry/dao/VertexReader.java | 4 +++- .../sunbirdrc/registry/service/ElasticSearchService.java | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java index 6a204f87f..fcae8748c 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/dao/VertexReader.java @@ -497,7 +497,9 @@ public JsonNode readInternal(Vertex rootVertex) throws Exception { private void expandReferenceNodes(ObjectNode rootNode) { rootNode.fields().forEachRemaining(entry -> { - if(entry.getValue().isValueNode() && entry.getValue().asText().startsWith(DID_TYPE)) { + // Regex pattern to check for a DID + String pattern = "^"+ DID_TYPE+":[^:]+:[^:]+"; + if(entry.getValue().isValueNode() && entry.getValue().asText().matches(pattern)) { String[] dids = entry.getValue().asText().split(":"); String osid = RecordIdentifier.parse(dids[2]).getUuid(); Iterator vertexIterator = graph.traversal().clone().V().hasLabel(dids[1]).has(uuidPropertyName, osid); diff --git a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java index 5afa0ed43..a0359cce4 100644 --- a/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java +++ b/java/registry/src/main/java/dev/sunbirdrc/registry/service/ElasticSearchService.java @@ -31,6 +31,7 @@ import dev.sunbirdrc.registry.middleware.util.Constants; import dev.sunbirdrc.registry.util.RecordIdentifier; +import static dev.sunbirdrc.registry.middleware.util.Constants.DID_TYPE; import static dev.sunbirdrc.registry.middleware.util.Constants.ENTITY_TYPE; /** @@ -126,7 +127,8 @@ private ArrayNode expandReference(JsonNode searchedNode) { for (JsonNode node : arrayNode) { ObjectNode objectNode = (ObjectNode) node; objectNode.fields().forEachRemaining(objectField -> { - if(objectField.getValue().asText().startsWith("did:")) { + String pattern = "^"+ DID_TYPE+":[^:]+:[^:]+"; + if(objectField.getValue().asText().matches(pattern)) { String[] referenceStrSplit = objectField.getValue().asText().split(":"); String indexName = referenceStrSplit[1].toLowerCase(); String osid = referenceStrSplit[2]; @@ -185,7 +187,7 @@ private SearchQuery getSearchQuery(String entityName, String osid) throws JsonPr private SearchQuery getSearchQuery(String entityName, List osids) throws JsonProcessingException { ArrayNode osidsArrayNode = JsonNodeFactory.instance.arrayNode(); for (String osid: osids) { - osidsArrayNode.add(osid); + osidsArrayNode.add("1-"+osid); } ObjectNode objectNode = JsonNodeFactory.instance.objectNode(); ObjectNode conditionNode = JsonNodeFactory.instance.objectNode();