chore: prepare 2.37.0 (#1584)

SwissDataScienceCenter · Jun 29, 2023 · 91ca8b1 · 91ca8b1
2 parents d2233be + e32897d
commit 91ca8b1
Show file tree

Hide file tree

Showing 172 changed files with 7,802 additions and 769 deletions.
diff --git a/README.md b/README.md
@@ -333,15 +333,23 @@ sequenceDiagram
     participant TriplesGenerator
     participant TokenRepository
     participant GitLab
+    participant TriplesStore
 
     EventLog ->> EventLog: sends PROJECT_SYNC
     activate EventLog
-    EventLog ->> TokenRepository: fetches access token
-    EventLog ->> GitLab: calls the Project Details
-    loop if there project path is NOT the same in EventLog and GitLab
-    EventLog ->> CommitEventService: sends COMMIT_SYNC for the new path
-    EventLog ->> TriplesGenerator: sends CLEAN_UP_REQUEST for the old path
-    end
+      EventLog ->> TokenRepository: fetches access token
+      EventLog ->> GitLab: calls the Project Details
+      loop if there project path is NOT the same in EventLog and GitLab
+        EventLog ->> CommitEventService: sends COMMIT_SYNC for the new path
+        EventLog ->> TriplesGenerator: sends CLEAN_UP_REQUEST for the old path
+      end
+      EventLog ->> TriplesGenerator: sends SYNC_REPO_METADATA
+      activate TriplesGenerator
+        TriplesGenerator ->> GitLab: fetches project metadata
+        TriplesGenerator ->> TriplesStore: fetches project metadata
+        TriplesGenerator ->> TriplesStore: sends update queries if values needs updating (not for visibility changes)
+        TriplesGenerator ->> EventLog: sends RedoProjectTransformation (only when visibility changes)
+      deactivate TriplesGenerator
     deactivate EventLog    
 ```
 

diff --git a/acceptance-tests/src/test/resources/application.conf b/acceptance-tests/src/test/resources/application.conf
@@ -13,6 +13,17 @@ global-commit-sync-delay-on-request = 0 minutes
 services {
   self {
     url = "http://localhost:9001"
+
+    hook-validation-cache {
+      evict-strategy = oldest
+      ignore-empty-values = true
+      ttl = 0s
+      clear-config {
+        type = periodic
+        maximum-size = 0
+        interval = 10min
+      }
+    }
   }
   gitlab {
     url = "http://localhost:2048"

diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/Project.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/Project.scala
@@ -27,18 +27,17 @@ import eu.timepit.refined.numeric.Positive
 import io.renku.graph.acceptancetests.data.Project._
 import io.renku.graph.model.entities.Project.ProjectMember
 import io.renku.graph.model.projects.{GitLabId, Name, Path}
-import io.renku.graph.model.testentities
+import io.renku.graph.model.{projects, testentities}
 import io.renku.tinytypes._
 import io.renku.tinytypes.constraints._
 
 import java.net.{MalformedURLException, URL}
-import java.time.Instant
 
 final case class Project(entitiesProject: testentities.RenkuProject,
                          id:              GitLabId,
                          maybeCreator:    Option[ProjectMember],
                          members:         NonEmptyList[ProjectMember],
-                         updatedAt:       DateUpdated,
+                         dateModified:    projects.DateModified,
                          urls:            Urls,
                          starsCount:      StarsCount,
                          permissions:     Permissions,
@@ -53,11 +52,6 @@ object Project {
   final class StarsCount private (val value: Int) extends AnyVal with IntTinyType
   implicit object StarsCount extends TinyTypeFactory[StarsCount](new StarsCount(_)) with NonNegativeInt[StarsCount]
 
-  final class DateUpdated private (val value: Instant) extends AnyVal with InstantTinyType
-  implicit object DateUpdated
-      extends TinyTypeFactory[DateUpdated](new DateUpdated(_))
-      with InstantNotInTheFuture[DateUpdated]
-
   sealed trait Permissions extends Product with Serializable
 
   object Permissions {

diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/package.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/package.scala
@@ -44,14 +44,14 @@ package object data extends TSData with ProjectFunctions {
       projectGen:   Gen[testentities.RenkuProject],
       commitsCount: CommitsCount = CommitsCount.one
   ): Gen[Project] = for {
-    project     <- projectGen
-    id          <- projectIds
-    members     <- projectMembers.toGeneratorOfNonEmptyList(max = 3)
-    updatedAt   <- timestamps(min = project.dateCreated.value, max = now).toGeneratorOf[DateUpdated]
-    urls        <- urlsObjects
-    starsCount  <- starsCounts
-    permissions <- permissionsObjects
-    statistics  <- statisticsObjects.map(_.copy(commitsCount = commitsCount))
+    project      <- projectGen
+    id           <- projectIds
+    members      <- projectMembers.toGeneratorOfNonEmptyList(max = 3)
+    dateModified <- timestamps(min = project.dateCreated.value, max = now).toGeneratorOf[projects.DateModified]
+    urls         <- urlsObjects
+    starsCount   <- starsCounts
+    permissions  <- permissionsObjects
+    statistics   <- statisticsObjects.map(_.copy(commitsCount = commitsCount))
     _ = if (project.members.nonEmpty)
           throw new Exception(show"Test project should not have members")
     _ = if (project.maybeCreator.flatMap(_.maybeGitLabId).nonEmpty)
@@ -60,7 +60,7 @@ package object data extends TSData with ProjectFunctions {
                   id,
                   maybeCreator = project.maybeCreator.map(_.to[ProjectMember]),
                   members,
-                  updatedAt,
+                  dateModified,
                   urls,
                   starsCount,
                   permissions,

diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/package.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/package.scala
@@ -40,18 +40,18 @@ package object knowledgegraph {
     import imageEncoders._
 
     json"""{
-    "identifier":  ${project.id.value}, 
-    "path":        ${project.path.value}, 
-    "name":        ${project.name.value},
-    "visibility":  ${project.entitiesProject.visibility.value},
-    "created":     ${(project.entitiesProject.dateCreated, project.entitiesProject.maybeCreator)},
-    "updatedAt":   ${project.updatedAt.value},
-    "urls":        ${project.urls.toJson},
-    "forking":     ${project.entitiesProject.forksCount -> project.entitiesProject},
-    "keywords":    ${project.entitiesProject.keywords.map(_.value).toList.sorted},
-    "starsCount":  ${project.starsCount.value},
-    "permissions": ${toJson(project.permissions)},
-    "images":      ${project.entitiesProject.images -> project.path},
+    "identifier":   ${project.id.value},
+    "path":         ${project.path.value},
+    "name":         ${project.name.value},
+    "visibility":   ${project.entitiesProject.visibility.value},
+    "created":      ${(project.entitiesProject.dateCreated, project.entitiesProject.maybeCreator)},
+    "dateModified": ${project.dateModified.value},
+    "urls":         ${project.urls.toJson},
+    "forking":      ${project.entitiesProject.forksCount -> project.entitiesProject},
+    "keywords":     ${project.entitiesProject.keywords.map(_.value).toList.sorted},
+    "starsCount":   ${project.starsCount.value},
+    "permissions":  ${toJson(project.permissions)},
+    "images":       ${project.entitiesProject.images -> project.path},
     "statistics": {
       "commitsCount":     ${project.statistics.commitsCount.value},
       "storageSize":      ${project.statistics.storageSize.value},

diff --git a/...tance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/JsonEncoders.scala b/...tance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/gitlab/JsonEncoders.scala
@@ -143,8 +143,9 @@ trait JsonEncoders {
           "star_count"          -> project.starsCount.value.asJson,
           "path_with_namespace" -> project.path.value.asJson,
           "created_at"          -> project.entitiesProject.dateCreated.value.asJson,
+          "updated_at"          -> project.entitiesProject.dateModified.value.asJson,
           "creator_id"          -> project.maybeCreator.map(_.gitLabId).asJson,
-          "last_activity_at"    -> project.updatedAt.value.asJson,
+          "last_activity_at"    -> project.dateModified.value.asJson,
           "permissions"         -> project.permissions.asJson,
           "statistics"          -> project.statistics.asJson,
           "forked_from_project" -> (project.entitiesProject match {

diff --git a/commit-event-service/Dockerfile b/commit-event-service/Dockerfile
@@ -10,7 +10,7 @@ COPY . .
 RUN export PATH="/usr/local/sbt/bin:$PATH" && \
     apk update && apk add --no-cache --virtual .build-dependencies bash wget tar git && \
     mkdir -p "/usr/local/sbt" && \
-    wget -qO  - "https://github.com/sbt/sbt/releases/download/v1.9.0/sbt-1.9.0.tgz" | tar xz -C /usr/local/sbt --strip-components=1 && \
+    wget -qO  - "https://github.com/sbt/sbt/releases/download/v1.9.1/sbt-1.9.1.tgz" | tar xz -C /usr/local/sbt --strip-components=1 && \
     sbt writeVersionToVersionSbt && \
     sbt writeVersionToVersionConf && \
     sbt "project commit-event-service" stage  && \

diff --git a/...ies-search/src/main/scala/io/renku/entities/searchgraphs/projects/ProjectSearchInfo.scala b/...ies-search/src/main/scala/io/renku/entities/searchgraphs/projects/ProjectSearchInfo.scala
@@ -29,6 +29,7 @@ private final case class ProjectSearchInfo(id:               projects.ResourceId
                                            path:             projects.Path,
                                            visibility:       projects.Visibility,
                                            dateCreated:      projects.DateCreated,
+                                           dateModified:     projects.DateModified,
                                            maybeCreator:     Option[PersonInfo],
                                            keywords:         List[projects.Keyword],
                                            maybeDescription: Option[projects.Description],
@@ -38,13 +39,24 @@ private final case class ProjectSearchInfo(id:               projects.ResourceId
 private object ProjectSearchInfo {
 
   implicit val show: Show[ProjectSearchInfo] = Show.show {
-    case ProjectSearchInfo(id, name, path, visibility, dateCreated, maybeCreator, keywords, maybeDescription, images) =>
+    case ProjectSearchInfo(id,
+                           name,
+                           path,
+                           visibility,
+                           dateCreated,
+                           dateModified,
+                           maybeCreator,
+                           keywords,
+                           maybeDescription,
+                           images
+        ) =>
       List(
         show"id = $id".some,
         show"name = $name".some,
         show"path = $path".some,
         show"visibility = $visibility".some,
         show"dateCreated = $dateCreated".some,
+        show"dateModified = $dateModified".some,
         maybeCreator.map(creator => show"creator = $creator"),
         keywords match {
           case Nil => None

diff --git a/...s-search/src/main/scala/io/renku/entities/searchgraphs/projects/SearchInfoExtractor.scala b/...s-search/src/main/scala/io/renku/entities/searchgraphs/projects/SearchInfoExtractor.scala
@@ -30,6 +30,7 @@ private object SearchInfoExtractor {
       project.path,
       project.visibility,
       project.dateCreated,
+      project.dateModified,
       project.maybeCreator.map(toPersonInfo),
       project.keywords.toList,
       project.maybeDescription,

diff --git a/...ies-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/Encoders.scala b/...ies-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/Encoders.scala
@@ -74,6 +74,7 @@ private object Encoders {
       searchInfoQuad(ProjectSearchInfoOntology.pathProperty.id, info.path.asObject).some,
       searchInfoQuad(ProjectSearchInfoOntology.visibilityProperty.id, info.visibility.asObject).some,
       searchInfoQuad(ProjectSearchInfoOntology.dateCreatedProperty.id, info.dateCreated.asObject).some,
+      searchInfoQuad(ProjectSearchInfoOntology.dateModifiedProperty.id, info.dateModified.asObject).some,
       maybeDescriptionQuad
     ).flatten ++ creatorQuads ++ keywordsQuads ++ imagesQuads
   }

diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/ontology.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/ontology.scala
@@ -26,14 +26,15 @@ import io.renku.jsonld.ontology._
 
 object ProjectSearchInfoOntology {
 
-  val nameProperty:        DataProperty.Def = Project.Ontology.nameProperty
-  val pathProperty:        DataProperty.Def = Project.Ontology.pathProperty
-  val visibilityProperty:  DataProperty.Def = Project.Ontology.visibilityProperty
-  val dateCreatedProperty: DataProperty.Def = Project.Ontology.dateCreatedProperty
-  val keywordsProperty:    DataProperty.Def = Project.Ontology.keywordsProperty
-  val descriptionProperty: DataProperty.Def = Project.Ontology.descriptionProperty
-  val creatorProperty:     Property         = Project.Ontology.creator
-  val imageProperty:       Property         = Project.Ontology.image
+  val nameProperty:         DataProperty.Def = Project.Ontology.nameProperty
+  val pathProperty:         DataProperty.Def = Project.Ontology.pathProperty
+  val visibilityProperty:   DataProperty.Def = Project.Ontology.visibilityProperty
+  val dateCreatedProperty:  DataProperty.Def = Project.Ontology.dateCreatedProperty
+  val dateModifiedProperty: DataProperty.Def = Project.Ontology.dateModifiedProperty
+  val keywordsProperty:     DataProperty.Def = Project.Ontology.keywordsProperty
+  val descriptionProperty:  DataProperty.Def = Project.Ontology.descriptionProperty
+  val creatorProperty:      Property         = Project.Ontology.creator
+  val imageProperty:        Property         = Project.Ontology.image
 
   lazy val typeDef: Type = Type.Def(
     Class(renku / "DiscoverableProject"),
@@ -46,6 +47,7 @@ object ProjectSearchInfoOntology {
       pathProperty,
       visibilityProperty,
       dateCreatedProperty,
+      dateModifiedProperty,
       keywordsProperty,
       descriptionProperty
     )

diff --git a/entities-search/src/test/scala/io/renku/entities/search/EntitiesFinderOldSpec.scala b/entities-search/src/test/scala/io/renku/entities/search/EntitiesFinderOldSpec.scala
@@ -32,11 +32,11 @@ import io.renku.generators.Generators.Implicits._
 import io.renku.generators.Generators._
 import io.renku.graph.model._
 import io.renku.graph.model.projects.Visibility
-import io.renku.graph.model.testentities.{Dataset, RenkuProject, StepPlan}
 import io.renku.graph.model.testentities.generators.EntitiesGenerators
-import io.renku.http.rest.{SortBy, Sorting}
+import io.renku.graph.model.testentities.{Dataset, RenkuProject, StepPlan}
 import io.renku.http.rest.paging.PagingRequest
 import io.renku.http.rest.paging.model._
+import io.renku.http.rest.{SortBy, Sorting}
 import io.renku.testtools.IOSpec
 import io.renku.triplesstore._
 import org.scalatest.matchers.should
@@ -616,6 +616,7 @@ class EntitiesFinderOldSpec
       val projectDateCreated = timestamps(min = sinceAsInstant, max = Instant.now()).generateAs[projects.DateCreated]
       val ds -> project = renkuProjectEntities(visibilityPublic)
         .modify(replaceProjectDateCreated(to = projectDateCreated))
+        .modify(replaceProjectDateModified(projectModifiedDates(projectDateCreated.value).generateOne))
         .withActivities(
           activityEntities(
             stepPlanEntities().map(
@@ -657,6 +658,7 @@ class EntitiesFinderOldSpec
       val projectDateCreated = timestamps(max = sinceAsInstant minus (2, DAYS)).generateAs(projects.DateCreated)
       val _ -> project = renkuProjectEntities(visibilityPublic)
         .modify(replaceProjectDateCreated(to = projectDateCreated))
+        .modify(replaceProjectDateModified(projectModifiedDates(projectDateCreated.value).generateOne))
         .withActivities(
           activityEntities(
             stepPlanEntities().map(
@@ -704,12 +706,10 @@ class EntitiesFinderOldSpec
       val since          = sinceParams.generateOne
       val sinceAsInstant = Instant.from(since.value atStartOfDay ZoneOffset.UTC)
 
+      val projectDateCreated = timestamps(max = sinceAsInstant minus (2, DAYS)).generateAs(projects.DateCreated)
       val matchingDS -> dsProject = renkuProjectEntities(visibilityPublic)
-        .modify(
-          replaceProjectDateCreated(to =
-            timestamps(max = sinceAsInstant minus (2, DAYS)).generateAs(projects.DateCreated)
-          )
-        )
+        .modify(replaceProjectDateCreated(projectDateCreated))
+        .modify(replaceProjectDateModified(projectModifiedDates(projectDateCreated.value).generateOne))
         .addDataset(
           datasetEntities(provenanceImportedExternal)
             .modify(
@@ -745,6 +745,7 @@ class EntitiesFinderOldSpec
       val projectDateCreated = timestamps(max = untilAsInstant).generateAs[projects.DateCreated]
       val ds -> project = renkuProjectEntities(visibilityPublic)
         .modify(replaceProjectDateCreated(to = projectDateCreated))
+        .modify(replaceProjectDateModified(projectModifiedDates(projectDateCreated.value).generateOne))
         .withActivities(
           activityEntities(
             stepPlanEntities().map(
@@ -787,6 +788,7 @@ class EntitiesFinderOldSpec
         .generateAs(projects.DateCreated)
       val _ -> project = renkuProjectEntities(visibilityPublic)
         .modify(replaceProjectDateCreated(to = projectDateCreated))
+        .modify(replaceProjectDateModified(projectModifiedDates(projectDateCreated.value).generateOne))
         .withActivities(
           activityEntities(
             stepPlanEntities().map(
@@ -832,12 +834,11 @@ class EntitiesFinderOldSpec
       val until          = Until(untilParams.generateOne.value minusDays 2)
       val untilAsInstant = Instant.from(until.value atStartOfDay ZoneOffset.UTC)
 
+      val projectDateCreated =
+        timestampsNotInTheFuture(butYoungerThan = untilAsInstant plus (1, DAYS)).generateAs(projects.DateCreated)
       val matchingDS -> dsProject = renkuProjectEntities(visibilityPublic)
-        .modify(
-          replaceProjectDateCreated(to =
-            timestampsNotInTheFuture(butYoungerThan = untilAsInstant plus (1, DAYS)).generateAs(projects.DateCreated)
-          )
-        )
+        .modify(replaceProjectDateCreated(projectDateCreated))
+        .modify(replaceProjectDateModified(projectModifiedDates(projectDateCreated.value).generateOne))
         .addDataset(
           datasetEntities(provenanceImportedExternal)
             .modify(
@@ -879,6 +880,7 @@ class EntitiesFinderOldSpec
       val projectDateCreated = timestamps(min = sinceAsInstant, max = untilAsInstant).generateAs[projects.DateCreated]
       val dsInternal -> dsExternal -> _ -> _ -> project = renkuProjectEntities(visibilityPublic)
         .modify(replaceProjectDateCreated(to = projectDateCreated))
+        .modify(replaceProjectDateModified(projectModifiedDates(projectDateCreated.value).generateOne))
         .withActivities(
           activityEntities(
             stepPlanEntities().map(