diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index 8904a86da38..49981379f2c 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -237,7 +237,7 @@ class ServiceGetV2(BaseModel): access_rights: dict[GroupID, ServiceGroupAccessRightsV2] | None - classifiers: list[str] | None = None + classifiers: list[str] | None = [] quality: dict[str, Any] = {} history: list[ServiceRelease] = Field( diff --git a/packages/models-library/src/models_library/services_history.py b/packages/models-library/src/models_library/services_history.py index 43dbc3c9dbd..774213ad0d6 100644 --- a/packages/models-library/src/models_library/services_history.py +++ b/packages/models-library/src/models_library/services_history.py @@ -1,6 +1,7 @@ from datetime import datetime from typing import Any, ClassVar, TypeAlias +from models_library.services_base import ServiceKeyVersion from pydantic import BaseModel, Field from .services_types import ServiceVersion @@ -9,7 +10,7 @@ class Compatibility(BaseModel): # NOTE: as an object it is more maintainable than a list - can_update_to: ServiceVersion = Field( + can_update_to: ServiceVersion | ServiceKeyVersion = Field( ..., description="Latest compatible version at this moment." "Current service can update to this version and still work", diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/d0e56c2d0a0d_new_services_comp_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/d0e56c2d0a0d_new_services_comp_table.py new file mode 100644 index 00000000000..160919d3e9c --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/d0e56c2d0a0d_new_services_comp_table.py @@ -0,0 +1,97 @@ +"""new services_comp table + +Revision ID: d0e56c2d0a0d +Revises: 19f3d9085636 +Create Date: 2024-07-17 16:15:49.970615+00:00 + +""" +from typing import Final + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "d0e56c2d0a0d" +down_revision = "19f3d9085636" +branch_labels = None +depends_on = None + +# auto-update modified +# TRIGGERS ------------------------ +_TABLE_NAME: Final[str] = "services_compatibility" +_TRIGGER_NAME: Final[str] = "trigger_auto_update" # NOTE: scoped on table +_PROCEDURE_NAME: Final[ + str +] = f"{_TABLE_NAME}_auto_update_modified()" # NOTE: scoped on database +modified_timestamp_trigger = sa.DDL( + f""" +DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME}; +CREATE TRIGGER {_TRIGGER_NAME} +BEFORE INSERT OR UPDATE ON {_TABLE_NAME} +FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME}; + """ +) + +# PROCEDURES ------------------------ +update_modified_timestamp_procedure = sa.DDL( + f""" +CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME} +RETURNS TRIGGER AS $$ +BEGIN + NEW.modified := current_timestamp; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + """ +) + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "services_compatibility", + sa.Column("key", sa.String(), nullable=False), + sa.Column("version", sa.String(), nullable=False), + sa.Column( + "custom_policy", postgresql.JSONB(astext_type=sa.Text()), nullable=False + ), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("modified_by", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["key", "version"], + ["services_meta_data.key", "services_meta_data.version"], + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["modified_by"], ["users.id"], onupdate="CASCADE", ondelete="SET NULL" + ), + sa.PrimaryKeyConstraint("key", "version", name="services_compatibility_pk"), + ) + # ### end Alembic commands ### + + # custom + op.execute(update_modified_timestamp_procedure) + op.execute(modified_timestamp_trigger) + + +def downgrade(): + # custom + op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};") + op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};") + + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("services_compatibility") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/_common.py b/packages/postgres-database/src/simcore_postgres_database/models/_common.py index 4fd666b836a..c3f671d244a 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/_common.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/_common.py @@ -26,6 +26,38 @@ def column_modified_datetime(*, timezone: bool = True) -> sa.Column: ) +def column_created_by_user( + *, users_table: sa.Table, required: bool = False +) -> sa.Column: + return sa.Column( + "created_by", + sa.Integer, + sa.ForeignKey( + users_table.c.id, + onupdate="CASCADE", + ondelete="SET NULL", + ), + nullable=not required, + doc="Who created this row at `created`", + ) + + +def column_modified_by_user( + *, users_table: sa.Table, required: bool = False +) -> sa.Column: + return sa.Column( + "modified_by", + sa.Integer, + sa.ForeignKey( + users_table.c.id, + onupdate="CASCADE", + ondelete="SET NULL", + ), + nullable=not required, + doc="Who modified this row at `modified`", + ) + + _TRIGGER_NAME: Final[str] = "auto_update_modified_timestamp" diff --git a/packages/postgres-database/src/simcore_postgres_database/models/services.py b/packages/postgres-database/src/simcore_postgres_database/models/services.py index a3fb9ac68ea..3839667d8a1 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/services.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/services.py @@ -4,6 +4,7 @@ - Services have a key, version, and access rights defined by group ids """ + import sqlalchemy as sa from sqlalchemy import null from sqlalchemy.dialects.postgresql import ARRAY, JSONB @@ -11,13 +12,13 @@ from .base import metadata -# -# Combines properties as -# - service identifier: key, version -# - overridable properties of the service metadata defined upon publication (injected in the image labels) -# - extra properties assigned during its lifetime (e.g. deprecated, quality, etc) - services_meta_data = sa.Table( + # + # Combines properties as + # - service identifier: key, version + # - overridable properties of the service metadata defined upon publication (injected in the image labels) + # - extra properties assigned during its lifetime (e.g. deprecated, quality, etc) + # "services_meta_data", metadata, sa.Column( @@ -103,12 +104,11 @@ ) -# -# Defines access rights (execute_access, write_access) on a service (key) -# for a given group (gid) on a product (project_name) -# - services_access_rights = sa.Table( + # + # Defines access rights (execute_access, write_access) on a service (key) + # for a given group (gid) on a product (project_name) + # "services_access_rights", metadata, sa.Column( diff --git a/packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py b/packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py new file mode 100644 index 00000000000..aa3929385e3 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py @@ -0,0 +1,68 @@ +""" Services table + + - List of 3rd party services in the framework + - Services have a key, version, and access rights defined by group ids +""" + + +import sqlalchemy as sa +import typing_extensions +from sqlalchemy.dialects.postgresql import JSONB +from typing_extensions import NotRequired, Required + +from ._common import ( + column_created_datetime, + column_modified_by_user, + column_modified_datetime, +) +from .base import metadata +from .users import users + + +class CompatiblePolicyDict(typing_extensions.TypedDict, total=False): + # SpecifierSet e.g. ~=0.9 + # SEE https://packaging.python.org/en/latest/specifications/version-specifiers/#id5 + versions_specifier: Required[str] + # Only necessary if key!=PolicySpecifierDict.key + other_service_key: NotRequired[str | None] + + +services_compatibility = sa.Table( + # + # CUSTOM COMPATIBILITY POLICIES + # Otherwise default compatibility policy is employed. + # + "services_compatibility", + metadata, + sa.Column( + "key", + sa.String, + nullable=False, + doc="Service Key Identifier", + ), + sa.Column( + "version", + sa.String, + nullable=False, + doc="Service version", + ), + sa.Column( + "custom_policy", + JSONB, + nullable=False, + doc="PolicySpecifierDict with custom policy", + ), + # Traceability, i.e. when + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), + # Traceability, i.e. who + column_modified_by_user(users_table=users, required=True), + # Constraints + sa.ForeignKeyConstraint( + ["key", "version"], + ["services_meta_data.key", "services_meta_data.version"], + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("key", "version", name="services_compatibility_pk"), +) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py b/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py index 5e822ae6aad..3a72d803a19 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py @@ -7,7 +7,13 @@ from sqlalchemy.sql.expression import func from sqlalchemy.sql.selectable import Select -from ..tables import services_access_rights, services_meta_data, user_to_groups, users +from ..tables import ( + services_access_rights, + services_compatibility, + services_meta_data, + user_to_groups, + users, +) def list_services_stmt( @@ -172,6 +178,7 @@ def list_latest_services_with_history_stmt( services_meta_data.c.version, services_meta_data.c.deprecated, services_meta_data.c.created, + services_compatibility.c.custom_policy, # CompatiblePolicyDict | None ) .select_from( services_meta_data.join( @@ -184,11 +191,17 @@ def list_latest_services_with_history_stmt( (services_meta_data.c.key == services_access_rights.c.key) & (services_meta_data.c.version == services_access_rights.c.version) & (services_access_rights.c.product_name == product_name), - ).join( + ) + .join( user_to_groups, (user_to_groups.c.gid == services_access_rights.c.gid) & (user_to_groups.c.uid == user_id), ) + .outerjoin( + services_compatibility, + (services_meta_data.c.key == services_compatibility.c.key) + & (services_meta_data.c.version == services_compatibility.c.version), + ) ) .where(access_rights) .order_by( @@ -224,6 +237,8 @@ def list_latest_services_with_history_stmt( history_subquery.c.deprecated, "created", history_subquery.c.created, + "compatibility_policy", # NOTE: this is the `policy` + history_subquery.c.custom_policy, ) ).label("history"), ) @@ -316,6 +331,7 @@ def get_service_history_stmt( services_meta_data.c.version, services_meta_data.c.deprecated, services_meta_data.c.created, + services_compatibility.c.custom_policy, # CompatiblePolicyDict | None ) .select_from( # joins because access-rights might change per version @@ -323,10 +339,16 @@ def get_service_history_stmt( services_access_rights, (services_meta_data.c.key == services_access_rights.c.key) & (services_meta_data.c.version == services_access_rights.c.version), - ).join( + ) + .join( user_to_groups, (user_to_groups.c.gid == services_access_rights.c.gid), ) + .outerjoin( + services_compatibility, + (services_meta_data.c.key == services_compatibility.c.key) + & (services_meta_data.c.version == services_compatibility.c.version), + ) ) .where( (services_meta_data.c.key == service_key) @@ -338,18 +360,24 @@ def get_service_history_stmt( services_meta_data.c.key, sa.desc(_version(services_meta_data.c.version)), # latest version first ) - .subquery() + .alias("history_subquery") ) - return sa.select( - array_agg( - func.json_build_object( - "version", - history_subquery.c.version, - "deprecated", - history_subquery.c.deprecated, - "created", - history_subquery.c.created, - ) - ).label("history"), - ).group_by(history_subquery.c.key) + return ( + sa.select( + array_agg( + func.json_build_object( + "version", + history_subquery.c.version, + "deprecated", + history_subquery.c.deprecated, + "created", + history_subquery.c.created, + "compatibility_policy", # NOTE: this is the `policy` + history_subquery.c.custom_policy, + ) + ).label("history"), + ) + .select_from(history_subquery) + .group_by(history_subquery.c.key) + ) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index 01a07e61c88..aa1558a6b48 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -15,7 +15,7 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.users import GroupID, UserID from psycopg2.errors import ForeignKeyViolation -from pydantic import PositiveInt, ValidationError +from pydantic import PositiveInt, ValidationError, parse_obj_as from simcore_postgres_database.utils_services import create_select_latest_services_query from sqlalchemy import literal_column from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -23,6 +23,7 @@ from sqlalchemy.sql.expression import tuple_ from ...models.services_db import ( + ReleaseFromDB, ServiceAccessRightsAtDB, ServiceMetaDataAtDB, ServiceWithHistoryFromDB, @@ -383,6 +384,27 @@ async def list_latest_services( return (total_count, items_page) + async def get_service_history( + self, + # access-rights + product_name: ProductName, + user_id: UserID, + # get args + key: ServiceKey, + ) -> list[ReleaseFromDB] | None: + + stmt_history = get_service_history_stmt( + product_name=product_name, + user_id=user_id, + access_rights=AccessRightsClauses.can_read, + service_key=key, + ) + async with self.db_engine.begin() as conn: + result = await conn.execute(stmt_history) + row = result.one_or_none() + + return parse_obj_as(list[ReleaseFromDB], row.history) if row else None + # Service Access Rights ---- async def get_service_access_rights( diff --git a/services/catalog/src/simcore_service_catalog/db/tables.py b/services/catalog/src/simcore_service_catalog/db/tables.py index c2cc8617012..aaff065fcf7 100644 --- a/services/catalog/src/simcore_service_catalog/db/tables.py +++ b/services/catalog/src/simcore_service_catalog/db/tables.py @@ -4,19 +4,23 @@ services_access_rights, services_meta_data, ) +from simcore_postgres_database.models.services_compatibility import ( + services_compatibility, +) from simcore_postgres_database.models.services_specifications import ( services_specifications, ) from simcore_postgres_database.models.users import users __all__ = ( - "services_meta_data", - "services_access_rights", - "services_specifications", - "users", - "user_to_groups", "groups", "GroupType", "projects", "ProjectType", + "services_access_rights", + "services_compatibility", + "services_meta_data", + "services_specifications", + "user_to_groups", + "users", ) diff --git a/services/catalog/src/simcore_service_catalog/models/services_db.py b/services/catalog/src/simcore_service_catalog/models/services_db.py index 7371dae215d..0b3c556b57b 100644 --- a/services/catalog/src/simcore_service_catalog/models/services_db.py +++ b/services/catalog/src/simcore_service_catalog/models/services_db.py @@ -3,11 +3,11 @@ from models_library.services_access import ServiceGroupAccessRights from models_library.services_base import ServiceKeyVersion -from models_library.services_history import ServiceRelease from models_library.services_metadata_editable import ServiceMetaDataEditable from models_library.services_types import ServiceKey, ServiceVersion from pydantic import BaseModel, Field from pydantic.types import PositiveInt +from simcore_postgres_database.models.services_compatibility import CompatiblePolicyDict # ------------------------------------------------------------------- # Databases models @@ -54,17 +54,11 @@ class Config: } -class HistoryItem(BaseModel): +class ReleaseFromDB(BaseModel): version: ServiceVersion deprecated: datetime | None created: datetime - - def to_api_model(self) -> ServiceRelease: - return ServiceRelease.construct( - version=self.version, - released=self.created, - retired=self.deprecated, - ) + compatibility_policy: CompatiblePolicyDict | None class ServiceWithHistoryFromDB(BaseModel): @@ -84,11 +78,13 @@ class ServiceWithHistoryFromDB(BaseModel): modified: datetime deprecated: datetime | None # releases - history: list[HistoryItem] + history: list[ReleaseFromDB] -assert set(HistoryItem.__fields__).issubset( # nosec - set(ServiceWithHistoryFromDB.__fields__) +assert ( # nosec + set(ReleaseFromDB.__fields__) + .difference({"compatibility_policy"}) + .issubset(set(ServiceWithHistoryFromDB.__fields__)) ) diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/services/compatibility.py new file mode 100644 index 00000000000..20833383551 --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/services/compatibility.py @@ -0,0 +1,118 @@ +""" Manages service compatibility policies + +""" + +from models_library.basic_types import VersionStr +from models_library.products import ProductName +from models_library.services_base import ServiceKeyVersion +from models_library.services_history import Compatibility +from models_library.services_types import ServiceKey, ServiceVersion +from models_library.users import UserID +from packaging.specifiers import SpecifierSet +from packaging.version import Version +from simcore_service_catalog.utils.versioning import as_version + +from ..db.repositories.services import ServicesRepository +from ..models.services_db import ReleaseFromDB + + +def _get_default_compatibility_specs(target: ServiceVersion | Version) -> SpecifierSet: + """Default policy: + A version is compatible with target X.Y.Z if `>X.Y.Z, ~=X.Y.Z` (i.e. any patch released newer than the target) + SEE https://packaging.python.org/en/latest/specifications/version-specifiers/#id5 + """ + version = as_version(target) + return SpecifierSet( + f">{version}, ~={version.major}.{version.minor}.{version.micro}" + ) + + +def _get_latest_compatible_version( + target: ServiceVersion | Version, + service_versions: list[Version], + compatibility_specs: SpecifierSet | None = None, +) -> Version | None: + """ + Returns latest version in history that satisfies `>X.Y.Z, ~=X.Y.Z` (default policy if compatibility_specs=None) or compatibility_specs + Returns None if no version in history satisfies specs. + """ + compatibility_specs = compatibility_specs or _get_default_compatibility_specs( + target + ) + compatible_versions = [v for v in service_versions if v in compatibility_specs] + return max(compatible_versions, default=None) + + +def _convert_to_versions(service_history: list[ReleaseFromDB]) -> list[Version]: + return sorted( + (as_version(h.version) for h in service_history if not h.deprecated), + reverse=True, # latest first + ) + + +async def _evaluate_custom_compatibility( + repo: ServicesRepository, + product_name: ProductName, + user_id: UserID, + target_version: ServiceVersion, + released_versions: list[Version], + compatibility_policy: dict, +) -> Compatibility | None: + other_service_key = compatibility_policy.get("other_service_key") + other_service_versions = [] + + if other_service_key and ( + other_service_history := await repo.get_service_history( + product_name=product_name, + user_id=user_id, + key=ServiceKey(other_service_key), + ) + ): + other_service_versions = _convert_to_versions(other_service_history) + + versions_specifier = SpecifierSet(compatibility_policy["versions_specifier"]) + versions_to_check = other_service_versions or released_versions + + if latest_version := _get_latest_compatible_version( + target_version, versions_to_check, versions_specifier + ): + if other_service_key: + return Compatibility( + can_update_to=ServiceKeyVersion( + key=other_service_key, version=VersionStr(latest_version) + ) + ) + return Compatibility(can_update_to=ServiceKey(f"{latest_version}")) + + return None + + +async def evaluate_service_compatibility_map( + repo: ServicesRepository, + product_name: ProductName, + user_id: UserID, + service_release_history: list[ReleaseFromDB], +) -> dict[ServiceVersion, Compatibility]: + released_versions = _convert_to_versions(service_release_history) + result = {} + + for release in service_release_history: + compatibility = None + if release.compatibility_policy: + compatibility = await _evaluate_custom_compatibility( + product_name=product_name, + user_id=user_id, + repo=repo, + target_version=release.version, + released_versions=released_versions, + compatibility_policy=release.compatibility_policy, + ) + elif latest_version := _get_latest_compatible_version( + release.version, + released_versions, + ): + compatibility = Compatibility(can_update_to=ServiceKey(f"{latest_version}")) + + result[release.version] = compatibility + + return result diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 28fd0ebe347..32d858b4fe3 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -8,6 +8,7 @@ from models_library.products import ProductName from models_library.rest_pagination import PageLimitInt from models_library.services_enums import ServiceType +from models_library.services_history import Compatibility, ServiceRelease from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID @@ -25,6 +26,7 @@ from simcore_service_catalog.services.director import DirectorApi from ..db.repositories.services import ServicesRepository +from .compatibility import evaluate_service_compatibility_map from .function_services import is_function_service _logger = logging.getLogger(__name__) @@ -42,10 +44,13 @@ def _db_to_api_model( service_db: ServiceWithHistoryFromDB, access_rights_db: list[ServiceAccessRightsAtDB], service_manifest: ServiceMetaDataPublished, + compatibility_map: dict[ServiceKey, Compatibility] | None = None, ) -> ServiceGetV2: - assert ( + compatibility_map = compatibility_map or {} + assert ( # nosec _deduce_service_type_from(service_db.key) == service_manifest.service_type - ) # nosec + ) + return ServiceGetV2( key=service_db.key, version=service_db.version, @@ -70,7 +75,15 @@ def _db_to_api_model( }, classifiers=service_db.classifiers, quality=service_db.quality, - history=[h.to_api_model() for h in service_db.history], + history=[ + ServiceRelease.construct( + version=h.version, + released=h.created, + retired=h.deprecated, + compatibility=compatibility_map.get(h.version), + ) + for h in service_db.history + ], ) @@ -110,13 +123,24 @@ async def list_services_paginated( (s.key, s.version): s for s in got if isinstance(s, ServiceMetaDataPublished) } - # NOTE: aggregates published (i.e. not editable) is still missing in this version items = [ - _db_to_api_model(s, ar, sm) + _db_to_api_model( + service_db=s, access_rights_db=ar, service_manifest=sm, compatibility_map=cm + ) for s in services if ( (ar := access_rights.get((s.key, s.version))) and (sm := service_manifest.get((s.key, s.version))) + and ( + # NOTE: This operation might be resource-intensive. + # It is temporarily implemented on a trial basis. + cm := await evaluate_service_compatibility_map( + repo, + product_name=product_name, + user_id=user_id, + service_release_history=s.history, + ) + ) ) ] @@ -167,7 +191,14 @@ async def get_service( director_client=director_api, ) - return _db_to_api_model(service, access_rights, service_manifest) + compatibility_map = await evaluate_service_compatibility_map( + repo, + product_name=product_name, + user_id=user_id, + service_release_history=service.history, + ) + + return _db_to_api_model(service, access_rights, service_manifest, compatibility_map) async def update_service( diff --git a/services/catalog/tests/unit/test_services_compatibility.py b/services/catalog/tests/unit/test_services_compatibility.py new file mode 100644 index 00000000000..8592a80344f --- /dev/null +++ b/services/catalog/tests/unit/test_services_compatibility.py @@ -0,0 +1,283 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + + +import arrow +import pytest +from models_library.services_types import ServiceVersion +from models_library.users import UserID +from packaging.specifiers import SpecifierSet +from packaging.version import Version +from pytest_mock import MockerFixture, MockType +from simcore_service_catalog.db.repositories.services import ServicesRepository +from simcore_service_catalog.models.services_db import ReleaseFromDB +from simcore_service_catalog.services.compatibility import ( + _get_latest_compatible_version, + evaluate_service_compatibility_map, +) + +# References +# +# - Semantic versioning: +# - https://semver.org/ +# - Python Packaging User Guide: +# - https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers +# - `packaging` library +# - https://packaging.pypa.io/en/stable/version.html +# - https://packaging.pypa.io/en/stable/specifiers.html +# + + +def test_compatible_with_minor_release(): + """Testing https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release + + The following groups of version clauses are equivalent: + ~= 2.2 + >= 2.2, == 2.* + """ + minor_compatible_spec = SpecifierSet("~=2.2") + + assert "2.2" in minor_compatible_spec + assert "2.2.0" in minor_compatible_spec + + assert Version("2.2") == Version("2.2.0") + + # bigger patch -> compatible + assert "2.2.1" in minor_compatible_spec + + # bigger minor -> compatible + assert "2.3" in minor_compatible_spec + + # bigger major -> INcompatible + assert "3.3" not in minor_compatible_spec + + # smaller major -> INcompatible + assert "2.1" not in minor_compatible_spec + assert "1.0" not in minor_compatible_spec + assert "0.1.5" not in minor_compatible_spec + + +def test_compatible_with_patch_release(): + """Testing https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release + + The following groups of version clauses are equivalent: + ~= 1.4.5 + >= 1.4.5, == 1.4.* + """ + patch_compatible_spec = SpecifierSet("~=1.4.5") + + assert "1.4.5" in patch_compatible_spec + + # bigger patch -> compatible + assert "1.4.6" in patch_compatible_spec + + # smaller patch -> INcompatible + assert "1.4.4" not in patch_compatible_spec + + # bigger minor -> INcompatible! + assert "1.5" not in patch_compatible_spec + assert "1.5.1" not in patch_compatible_spec + + # smaller major -> INcompatible + assert "0.1.5" not in patch_compatible_spec + assert "1.0" not in patch_compatible_spec + assert "1.1" not in patch_compatible_spec + assert "1.3" not in patch_compatible_spec + + +@pytest.fixture +def versions_history() -> list[Version]: + return sorted( + Version(f"{M}.{m}.{p}") + for M in range(10) + for m in range(0, 5, 2) + for p in range(0, 10, 4) + ) + + +def test_version_specifiers(versions_history: list[Version]): + # given a list of versions, test the first compatibilty starting from the latest + # If i have ">1.2.23,~=1.2.23" + + version = Version("1.2.3") + + # >1.2.3 + newer_version_spec = SpecifierSet(f">{version}") + + # >= 1.2, == 1.* + minor_compatible_spec = SpecifierSet(f"~={version.major}.{version.minor}") + + # >= 1.2.3, == 1.2.* + patch_compatible_spec = SpecifierSet( + f"~={version.major}.{version.minor}.{version.micro}" + ) + + compatible = list( + (minor_compatible_spec & newer_version_spec).filter(versions_history) + ) + assert version not in compatible + assert all(v > version for v in compatible) + assert all(v.major == version.major for v in compatible) + + latest_compatible = compatible[-1] + assert version < latest_compatible + + compatible = list( + (patch_compatible_spec & newer_version_spec).filter(versions_history) + ) + assert version not in compatible + assert all(v > version for v in compatible) + assert all( + v.major == version.major and v.minor == version.minor for v in compatible + ) + latest_compatible = compatible[-1] + assert version < latest_compatible + + +def test_get_latest_compatible_version(versions_history: list[Version]): + latest_first_releases = sorted(versions_history, reverse=True) + + # cannot upgrde to anything + latest = latest_first_releases[0] + assert _get_latest_compatible_version(latest, latest_first_releases) is None + + # bump MAJOR + not_released = Version(f"{latest.major+1}") + assert _get_latest_compatible_version(not_released, latest_first_releases) is None + + # decrease patch + target = Version(f"{latest.major}.{latest.minor}.{latest.micro-1}") + assert _get_latest_compatible_version(target, latest_first_releases) == latest + + # decrease minor (with default compatibility specs) + target = Version(f"{latest.major}.{latest.minor-2}.0") + latest_compatible = _get_latest_compatible_version(target, latest_first_releases) + assert latest_compatible + assert latest_compatible < latest + + +def _create_as(cls, **overrides): + kwargs = { + "deprecated": None, + "created": arrow.now().datetime, + "compatibility_policy": None, + } + kwargs.update(overrides) + return cls(**kwargs) + + +@pytest.fixture +def mock_repo(mocker: MockerFixture) -> MockType: + return mocker.AsyncMock(ServicesRepository) + + +async def test_evaluate_service_compatibility_map_with_default_policy( + mock_repo: MockType, user_id: UserID +): + service_release_history = [ + _create_as(ReleaseFromDB, version="1.0.0"), + _create_as(ReleaseFromDB, version="1.0.1"), + _create_as(ReleaseFromDB, version="1.1.0"), + _create_as(ReleaseFromDB, version="2.0.0"), + ] + + compatibility_map = await evaluate_service_compatibility_map( + mock_repo, "product_name", user_id, service_release_history + ) + + assert len(compatibility_map) == 4 + assert compatibility_map[ServiceVersion("1.0.0")].can_update_to == "1.0.1" + assert compatibility_map[ServiceVersion("1.0.1")] is None + assert compatibility_map[ServiceVersion("1.1.0")] is None + assert compatibility_map[ServiceVersion("2.0.0")] is None + + +async def test_evaluate_service_compatibility_map_with_custom_policy( + mock_repo: MockType, user_id: UserID +): + service_release_history = [ + _create_as(ReleaseFromDB, version="1.0.0"), + _create_as( + ReleaseFromDB, + version="1.0.1", + compatibility_policy={"versions_specifier": ">1.1.0,<=2.0.0"}, + ), + _create_as(ReleaseFromDB, version="1.2.0"), + _create_as(ReleaseFromDB, version="2.0.0"), + ] + + compatibility_map = await evaluate_service_compatibility_map( + mock_repo, "product_name", user_id, service_release_history + ) + + assert len(compatibility_map) == 4 + assert ( + compatibility_map[ServiceVersion("1.0.0")].can_update_to == "1.0.1" + ) # default + assert ( + compatibility_map[ServiceVersion("1.0.1")].can_update_to == "2.0.0" + ) # version customized + assert compatibility_map[ServiceVersion("1.2.0")] is None + assert compatibility_map[ServiceVersion("2.0.0")] is None + + +async def test_evaluate_service_compatibility_map_with_other_service( + mock_repo: MockType, user_id: UserID +): + service_release_history = [ + _create_as(ReleaseFromDB, version="1.0.0"), + _create_as( + ReleaseFromDB, + version="1.0.1", + compatibility_policy={ + "other_service_key": "simcore/services/comp/other_service", + "versions_specifier": "<=5.1.0", + }, + ), + ] + + mock_repo.get_service_history.return_value = [ + _create_as(ReleaseFromDB, version="5.0.0"), + _create_as(ReleaseFromDB, version="5.1.0"), + _create_as(ReleaseFromDB, version="5.2.0"), + ] + + compatibility_map = await evaluate_service_compatibility_map( + mock_repo, "product_name", user_id, service_release_history + ) + + assert len(compatibility_map) == 2 + assert compatibility_map[ServiceVersion("1.0.0")].can_update_to == "1.0.1" + # NOTE: 1.0.1 is also upgradable but it is not evaluated as so because our algorithm only + # checks comptatibility once instead of recursively + + assert ( + compatibility_map[ServiceVersion("1.0.1")].can_update_to.key + == "simcore/services/comp/other_service" + ) + assert compatibility_map[ServiceVersion("1.0.1")].can_update_to.version == "5.1.0" + + +async def test_evaluate_service_compatibility_map_with_deprecated_versions( + mock_repo: MockType, user_id: UserID +): + service_release_history = [ + _create_as(ReleaseFromDB, version="1.0.0"), + _create_as(ReleaseFromDB, version="1.0.1", deprecated=True), + _create_as(ReleaseFromDB, version="1.2.0"), + _create_as(ReleaseFromDB, version="1.2.5"), + ] + + compatibility_map = await evaluate_service_compatibility_map( + mock_repo, "product_name", user_id, service_release_history + ) + + assert len(compatibility_map) == 4 + assert ( + compatibility_map[ServiceVersion("1.0.0")] is None + ) # cannot upgrade to deprecated 1.0.1 + assert compatibility_map[ServiceVersion("1.0.1")] is None # Deprecated version + assert compatibility_map[ServiceVersion("1.2.0")].can_update_to == "1.2.5" + assert compatibility_map[ServiceVersion("1.2.5")] is None diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 2a740f2cb06..c0f335781da 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3210,7 +3210,7 @@ paths: '403': description: ProjectInvalidRightsError '404': - description: ProjectNotFoundError, UserDefaultWalletNotFoundError + description: UserDefaultWalletNotFoundError, ProjectNotFoundError '409': description: ProjectTooManyProjectOpenedError '422': @@ -6060,8 +6060,10 @@ components: properties: canUpdateTo: title: Canupdateto - pattern: ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ - type: string + anyOf: + - pattern: ^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$ + type: string + - $ref: '#/components/schemas/ServiceKeyVersion' description: Latest compatible version at this moment.Current service can update to this version and still work ComputationStart: