diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index ee240cc2b..b82633eee 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -1,3 +1,4 @@ +--- name: Build, test and push a Docker image on: @@ -19,7 +20,10 @@ jobs: contents: read deployments: write id-token: write - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + env: + # Disable docker compose volume mounts in docker-compose.override.yml + COMPOSE_FILE: docker-compose.yml steps: - uses: actions/checkout@v4 - name: Get info @@ -37,22 +41,24 @@ jobs: "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > version.json - name: Output version.json run: cat version.json + - name: Install just + run: sudo apt-get update && sudo apt-get install -y just - name: Build Docker images - run: make build + run: | + just build + docker compose images - name: Verify requirements.txt contains correct dependencies run: | - docker compose run --rm --no-deps test-ci bash ./bin/run_verify_reqs.sh + just verify-reqs - name: Run lint check run: | - make .env - docker compose run --rm --no-deps test-ci bash ./bin/run_lint.sh - docker compose run --rm --no-deps frontend-ci lint + just lint - name: Run tests run: | - docker compose run --rm test-ci bash ./bin/run_test.sh + just test - name: Build docs run: | - docker compose run --rm --no-deps test-ci bash make -C docs/ html + just docs - name: Set Docker image tag to "latest" for updates of the main branch if: github.ref == 'refs/heads/main' diff --git a/Makefile b/Makefile deleted file mode 100644 index 39bdcdd4f..000000000 --- a/Makefile +++ /dev/null @@ -1,134 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Include my.env and export it so variables set in there are available -# in the Makefile. -include .env -export - -DOCKER := $(shell which docker) -DC=${DOCKER} compose - -SERVICES=db fakesentry redis-cache localstack statsd oidcprovider gcs-emulator - -.DEFAULT_GOAL := help -.PHONY: help -help: - @echo "Usage: make RULE" - @echo "" - @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' Makefile \ - | grep -v grep \ - | sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1\3/p' \ - | column -t -s '|' - @echo "" - @echo "Adjust your .env file to set configuration." - @echo "" - @echo "See https://tecken.readthedocs.io/ for more documentation." - -# Dev configuration steps -.docker-build: - make build - -.devcontainer-build: - make devcontainerbuild - -.env: - ./bin/cp-dist-file.sh docker/config/env-dist .env - -slick.sh: - ./bin/cp-dist-file.sh docker/config/slick.sh-dist slick.sh - chmod 755 slick.sh - -.PHONY: build -build: .env ## | Build docker images. - ${DC} build --progress plain base frontend fakesentry ${SERVICES} - touch .docker-build - -.PHONY: setup -setup: .env ## | Initialize services. - ${DC} run --rm web bash /app/bin/setup-services.sh - -.PHONY: run -run: .env .docker-build ## | Run the web app and services. - # NOTE(willkg): We tag all the services with --attach here to - # prevent dependencies from spamming stdout - ${DC} up \ - --attach web \ - --attach frontend \ - --attach fakesentry \ - web frontend fakesentry - -.PHONY: devcontainerbuild -devcontainerbuild: .env .docker-build ## | Build VS Code development container. - ${DC} build devcontainer - touch .devcontainer-build - -.PHONY: devcontainer -devcontainer: .env .docker-build .devcontainer-build ## | Run VS Code development container. - ${DC} up --detach devcontainer - -.PHONY: stop -stop: .env ## | Stop docker containers. - ${DC} stop - -.PHONY: shell -shell: .env .docker-build ## | Open a shell in web container. - ${DC} run --rm web bash - -.PHONY: clean -clean: .env stop ## | Stop and remove docker containers and artifacts. - ${DC} rm -f - rm -fr .docker-build - rm -rf frontend/build/ - git restore frontend/build/ - -.PHONY: clear-cache -clear-cache: ## | Clear Redis cache. - ${DC} run --rm redis-cache redis-cli -h redis-cache FLUSHDB - -.PHONY: redis-cache-cli -redis-cache-cli: .env .docker-build ## | Open Redis CLI to cache Redis server. - ${DC} run --rm redis-cache redis-cli -h redis-cache - -.PHONY: psql -psql: .env .docker-build ## | Open psql cli. - @echo "\e[0;32mNOTE: Use password 'postgres'.\e[0m\n" - ${DC} run --rm db psql -h db -U postgres -d tecken - -.PHONY: test -test: .env .docker-build ## | Run Python unit test suite. - ${DC} up --detach ${SERVICES} - ${DC} run --rm test bash ./bin/run_test.sh - -.PHONY: testshell -testshell: .env .docker-build ## | Open shell in test environment. - ${DC} up --detach ${SERVICES} - ${DC} run --rm test bash ./bin/run_test.sh --shell - -.PHONY: docs -docs: .env .docker-build ## | Build docs. - ${DC} run --rm --no-deps web bash make -C docs/ clean - ${DC} run --rm --no-deps web bash make -C docs/ html - -.PHONY: lint -lint: .env .docker-build ## | Lint code. - ${DC} run --rm --no-deps test bash ./bin/run_lint.sh - ${DC} run --rm frontend lint - -.PHONY: lintfix -lintfix: .env .docker-build ## | Reformat code. - ${DC} run --rm --no-deps test bash ./bin/run_lint.sh --fix - ${DC} run --rm frontend lintfix - -.PHONY: rebuildreqs -rebuildreqs: .env .docker-build ## | Rebuild requirements.txt file after requirements.in changes. - ${DC} run --rm --no-deps web bash pip-compile --generate-hashes - -.PHONY: updatereqs -updatereqs: .env .docker-build ## | Update deps in requirements.txt file. - ${DC} run --rm --no-deps web bash pip-compile --generate-hashes -U - -.PHONY: servicestatus -servicestatus: .env .docker-build ## | Update deps in requirements.txt file. - ${DC} run --rm --no-deps web bash service-status diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 9a6091426..d4bfda6d2 100755 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -30,7 +30,7 @@ shift case ${SERVICE} in web) ## Run Tecken web service - exec honcho -f /app//Procfile --no-prefix start + exec honcho -f /app/Procfile --no-prefix start ;; bash) ## Open a bash shell or run something else if [ -z "$*" ]; then diff --git a/bin/run_lint.sh b/bin/run_lint.sh index d3978c08b..61246adf0 100755 --- a/bin/run_lint.sh +++ b/bin/run_lint.sh @@ -16,7 +16,16 @@ FILES="bin tecken smoketests systemtests" PYTHON_VERSION=$(python --version) -if [[ $1 == "--fix" ]]; then +if [[ "${1:-}" == "--help" ]]; then + echo "Usage: $0 [OPTIONS]" + echo + echo " Lint code" + echo + echo "Options:" + echo " --help Show this message and exit." + echo " --fix Reformat code." + +elif [[ $1 == "--fix" ]]; then echo ">>> ruff fix (${PYTHON_VERSION})" ruff format $FILES ruff check --fix $FILES @@ -38,5 +47,6 @@ else license-check . fi - # NOTE(willkg): linting frontend files is done in another script + # NOTE(willkg): linting frontend files is done in another script because + # it has to run in the frontend container fi diff --git a/bin/run_web.sh b/bin/run_web.sh index 58046d5d6..98a4a058b 100755 --- a/bin/run_web.sh +++ b/bin/run_web.sh @@ -12,7 +12,7 @@ export PROCESS_NAME=webapp -if [ "$1" == "--dev" ]; then +if [ "$LOCAL_DEV_ENV" == "true" ]; then python manage.py migrate --noinput python manage.py runserver 0.0.0.0:${PORT} diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 000000000..bb59deb44 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,15 @@ +# define volumes in docker-compose.override.yml so that can be ignored in CI +--- +services: + base: + volumes: + - .:/app + web: + volumes: + - .:/app + test: + volumes: + - .:/app + frontend: + volumes: + - ./frontend:/app diff --git a/docker-compose.yml b/docker-compose.yml index 285a1224a..c7041e1df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,28 +29,8 @@ services: localstack: *condition_service_healthy oidcprovider: *condition_service_healthy statsd: *condition_service_healthy - volumes: - - $PWD:/app - command: ["web", "--dev"] - - # Container specifically for running tests. - test-ci: - extends: - service: base - env_file: - - docker/config/local_dev.env - - docker/config/test.env - depends_on: - db: *condition_service_healthy - gcs-emulator: *condition_service_healthy - fakesentry: *condition_service_healthy - redis-cache: *condition_service_healthy - localstack: *condition_service_healthy - oidcprovider: *condition_service_healthy - statsd: *condition_service_healthy + command: ["web"] - # TODO(2024-01-30): This configuration can be simplified using `extends: test-ci` once the fix for - # https://github.com/docker/compose/issues/11413 is available in all relevant environments. test: extends: service: base @@ -65,8 +45,6 @@ services: localstack: *condition_service_healthy oidcprovider: *condition_service_healthy statsd: *condition_service_healthy - volumes: - - $PWD:/app # Dev container https://containers.dev/, e.g. for VS Code devcontainer: @@ -105,25 +83,17 @@ services: redis-cache: *condition_service_healthy command: web - # Same as 'frontend' but no volumes or command - frontend-ci: + frontend: build: - context: . dockerfile: frontend/Dockerfile args: userid: ${USE_UID:-10001} groupid: ${USE_GID:-10001} - - frontend: - extends: - frontend-ci environment: - NODE_ENV=development ports: - "3000:3000" - "35729:35729" - volumes: - - $PWD/frontend:/app command: start # https://hub.docker.com/_/postgres/ diff --git a/docker/config/slick.sh-dist b/docker/config/slick.sh-dist index f4b0d9468..2b9643386 100644 --- a/docker/config/slick.sh-dist +++ b/docker/config/slick.sh-dist @@ -21,13 +21,10 @@ FAKETOKEN="c7c1f8cab79545b6a06bc4122f0eb3cb" FAKETOKEN_TRY="ebc447f574924b099e1be15527b62436" # Build Tecken -make build - -# Start services -docker compose up -d --remove-orphans db redis-cache statsd localstack oidcprovider fakesentry +just build # Run setup -make setup +just setup # Set up user docker compose exec oidcprovider /code/manage.py createuser "${FAKEUSERNAME}" "${FAKEPASSWORD}" "${FAKEEMAIL}" @@ -39,4 +36,4 @@ docker compose run --rm web bash python manage.py createtoken "--try-upload" "${ curl -X POST http://localhost:8090/api/flush/ # Reset Redis -make clear-cache +just clear-cache diff --git a/docs/dev.rst b/docs/dev.rst index 0c0924fb4..60251b3cc 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -9,7 +9,7 @@ Development Setup quickstart ================ -1. Install required software: Docker, make, and git. +1. Install required software: Docker, just, and git. **Linux**: @@ -19,17 +19,17 @@ Setup quickstart Install `Docker for Mac `_. - Use `homebrew `_ to install make and git: + Use `homebrew `_ to install just and git: .. code-block:: shell - $ brew install make git + $ brew install just git **Other**: Install `Docker `_. - Install `make `_. + Install `just `_. Install `git `_. @@ -45,13 +45,13 @@ Setup quickstart .. code-block:: shell - $ make .env + $ just _env Then edit the file and set the ``APP_UID`` and ``APP_GID`` variables. These will get used when creating the app user in the base image. If you ever want different values, change them in ``.env`` and re-run - ``make build``. + ``just build``. 4. Build Docker images for Socorro services. @@ -59,7 +59,7 @@ Setup quickstart .. code-block:: shell - $ make build + $ just build That will build the app Docker image required for development. @@ -69,7 +69,7 @@ Setup quickstart .. code-block:: shell - $ make setup + $ just setup This creates the Postgres database and sets up tables, integrity rules, and a bunch of other things. @@ -82,7 +82,7 @@ To run the webapp service, do: .. code-block:: shell - $ make run + $ just run The Symbols Service webapp is at: http://localhost:3000 @@ -108,7 +108,7 @@ first run: .. code-block:: shell - $ make devcontainerbuild + $ just build devcontainer What services are running in a local dev environment @@ -150,12 +150,12 @@ similar services, for example: EXPOSE_STATSD_PORT=8281 EXPOSE_GCS_EMULATOR_PORT=4443 -If you are using a development container for VS Code, you make need to restart +If you are using a development container for VS Code, you may need to restart the container to pick up changes: .. code-block:: shell - $ make devcontainer + $ just run devcontainer How to create a script to recreate your local dev environment @@ -165,7 +165,7 @@ Run: .. code-block:: shell - $ make slick.sh + $ just slick-sh Then edit the ``slick.sh`` script filling in: @@ -284,7 +284,7 @@ Do this: .. code-block:: shell - $ make shell + $ just shell app@xxx:/app$ ./manage.py makemigration --name "BUGID_desc" APP @@ -298,7 +298,7 @@ Use the ``bin/s3_cli.py`` script: .. code-block:: shell - $ make shell + $ just shell app@xxx:/app$ ./bin/s3_cli.py --help Usage: s3_cli.py [OPTIONS] COMMAND [ARGS]... @@ -324,7 +324,7 @@ Use the ``gcs-cli`` script: .. code-block:: shell - $ make shell + $ just shell app@5a9103973085:/app$ gcs-cli --help Usage: gcs-cli [OPTIONS] COMMAND [ARGS]... @@ -348,7 +348,7 @@ We use postgresql. To open a psql shell, do: .. code-block:: shell - $ make psql + $ just psql NOTE: Password is 'postgres'. /usr/bin/docker compose run --rm db psql -h db -U postgres -d tecken Password for user postgres: @@ -407,7 +407,7 @@ Pull requests Pull request summary should indicate the bug the pull request addresses. Use a hyphen between "bug" and the bug ID(s). For example:: - bug-nnnnnnn: removed frog from tree class + bug-nnnnnnn: removed frog from tree class For multiple bugs fixed within a single pull request, list the bugs out individually. For example:: @@ -476,14 +476,14 @@ To lint all the code, do: .. code-block:: bash - $ make lint + $ just lint To reformat all the code, do: .. code-block:: bash - $ make lintfix + $ just lint --fix HTML/CSS conventions @@ -535,13 +535,13 @@ To add a new dependency, add it to the file and then do: .. code-block:: shell - $ make rebuildreqs + $ just rebuild-reqs Then rebuild your docker environment: .. code-block:: shell - $ make build + $ just build If there are problems, it'll tell you. @@ -550,7 +550,7 @@ dependencies. To do this, run: .. code-block:: shell - $ make updatereqs + $ just updatereqs JavaScript dependencies (Symbols Service) @@ -592,7 +592,7 @@ To build the docs, do: .. code-block:: shell - $ make docs + $ just docs Then view ``docs/_build/html/index.html`` in your browser. @@ -609,7 +609,7 @@ To run the tests, do: .. code-block:: shell - $ make test + $ just test Tests for the Symbols Service webapp go in ``tecken/tests/``. @@ -618,7 +618,7 @@ the testshell: .. code-block:: shell - $ make testshell + $ just test-shell app@xxx:/app$ pytest @@ -691,12 +691,12 @@ Before we start doing local Upload By Download URL, you need to make your instance less secure since you'll be using URLs like ``http://localhost:9090``. Add ``DJANGO_ALLOW_UPLOAD_BY_ANY_DOMAIN=True`` to your ``.env`` file. -To serve them locally, first start the dev server (``make run``). Then +To serve them locally, first start the dev server (``just run``). Then you need to start a bash shell in the current running web container: .. code-block:: shell - $ make shell + $ just shell Now, you need some ``.zip`` files in the root of the project since it's mounted and can be seen by the containers. Once they're there, start a @@ -735,7 +735,7 @@ To simulate that Redis is "struggling" you can use the .. code-block:: shell - $ make redis-cache-cli + $ just redis-cache-cli redis-cache:6379> client pause 30000 OK diff --git a/docs/frontend.rst b/docs/frontend.rst index c0fb357ff..87dc28902 100644 --- a/docs/frontend.rst +++ b/docs/frontend.rst @@ -68,7 +68,7 @@ color coded and able to spit out any warnings or compilation errors. When run in docker, with non-TTY terminal, all output from the dev server is sent to ``stdout`` one line at a time. -When you start Docker for development (again ``make run`` or +When you start Docker for development (again ``just run`` or ``docker compose up web worker frontend``) it starts the dev server on port ``:3000`` and it also exposes a WebSocket on port ``:35729``. @@ -124,7 +124,7 @@ Watch out for ``node_modules``! If you ever run and build the frontend outside of Docker you end up with a directory ``frontend/node_modules`` which is ignored by git but is still part of the current working directory that Docker serves up and will -cause things like ``make build`` be excessively slow since the directory +cause things like ``just build`` be excessively slow since the directory can end up north of 100MB. If you have a ``frontend/node_modules`` directory, feel free to delete it. diff --git a/docs/redis.rst b/docs/redis.rst index 6edb7fab9..e8810f0e7 100644 --- a/docs/redis.rst +++ b/docs/redis.rst @@ -50,14 +50,14 @@ To go into CLI of the Redis database use this shortcuts: .. code-block:: shell - $ make redis-cache-cli + $ just redis-cache-cli From there you can see exactly what's stored. For example, to see the list of all symbols stored in the LRU cache: .. code-block:: shell - $ make redis-cache-cli + $ just redis-cache-cli redis-cache:6379> keys * 1) ":1:d0f08e5ed1882049b74f1962103df580" diff --git a/frontend/bin/build_yarn_lock.sh b/frontend/bin/build_yarn_lock.sh index 6c0793119..d925792a8 100755 --- a/frontend/bin/build_yarn_lock.sh +++ b/frontend/bin/build_yarn_lock.sh @@ -18,4 +18,4 @@ rm yarn.lock yarn install | cat rm -rf node_modules -echo "Run 'make build' now." +echo "Run 'just build' now." diff --git a/justfile b/justfile new file mode 100644 index 000000000..a3b87f07d --- /dev/null +++ b/justfile @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +_default: + @just --list + +_env: + #!/usr/bin/env sh + if [ ! -f .env ]; then + echo "Copying docker/config/env-dist to .env..." + ./bin/cp-dist-file.sh docker/config/env-dist .env + fi + + +SERVICES := "db redis-cache localstack statsd oidcprovider gcs-emulator" + +# Create a slick.sh file +slick-sh: + ./bin/cp-dist-file.sh docker/config/slick.sh-dist slick.sh + chmod 755 slick.sh + +# Build docker images. +build *args='base web frontend fakesentry db redis-cache localstack statsd oidcprovider gcs-emulator': _env + docker compose --progress plain build {{args}} + +# Set up services. +setup: _env + docker compose run --rm web bash ./bin/setup-services.sh + +# Run webapp and services. +run *args='--attach web --attach frontend --attach fakesentry web frontend fakesentry': _env + docker compose up {{args}} + +# Stop service containers. +stop *args: _env + docker compose stop {{args}} + +# Remove service containers and networks. +down *args: _env + docker compose down {{args}} + +# Open a shell in the web image. +shell *args='/bin/bash': _env + docker compose run --rm --entrypoint= web {{args}} + +# Open a shell in the test container. +test-shell *args='/bin/bash': _env + docker compose run --rm --entrypoint= test {{args}} + +# Stop and remove docker containers and artifacts. +clean: _env stop + docker compose rm -f + rm -fr .docker-build + rm -rf frontend/build/ + git restore frontend/build/ + +# Lint code, or use --fix to reformat Python code and apply auto-fixes for lint. +lint *args: _env + docker compose run --rm --no-deps test bash ./bin/run_lint.sh {{args}} + docker compose run --rm frontend lint + +# Run Python unit test suite. +test *args: _env + docker compose run --rm test bash ./bin/run_test.sh {{args}} + +# Build docs. +docs: _env + docker compose run --rm --no-deps web bash make -C docs/ clean + docker compose run --rm --no-deps web bash make -C docs/ html + +# Rebuild requirements.txt file after requirements.in changes. +rebuildreqs *args: _env + docker compose run --rm --no-deps web bash pip-compile --generate-hashes {{args}} + +# Verify that the requirements file is built by the version of Python that runs in the container. +verify-reqs: _env + docker compose run --rm --no-deps web bash ./bin/run_verify_reqs.sh + +# Check how far behind different server environments are from main tip. +service-status *args: _env + docker compose run --rm --no-deps web bash service-status {{args}} + +# Open psql cli. +psql *args: _env + @echo "\e[0;32mNOTE: Use password 'postgres'.\e[0m\n" + docker compose run --rm db psql -h db -U postgres -d tecken + +# Clear Redis cache. +clear-cache: _env + docker compose run --rm redis-cache redis-cli -h redis-cache FLUSHDB + +# Redis cli. +redis-cache-cli: _env + docker compose run --rm redis-cache redis-cli -h redis-cache diff --git a/smoketests/README.rst b/smoketests/README.rst index 7d1a94119..1e1117796 100644 --- a/smoketests/README.rst +++ b/smoketests/README.rst @@ -19,7 +19,7 @@ Before running the smoke tests, you need to `build a local dev environment Then set up the tests this way:: - $ make shell + $ just shell root@e62fb7ae586f:/app# cd smoketests root@e62fb7ae586f:/app/smoketests# ./setup_tests.py @@ -35,7 +35,7 @@ Running tests The smoke tests are run using the ``test_env.py`` Python script. You can get help about the command-line invocation of that script using:: - $ make shell + $ just shell root@e62fb7ae586f:/app# cd smoketests root@e62fb7ae586f:/app/smoketests# ./test_env.py --help diff --git a/smoketests/test_env.py b/smoketests/test_env.py index 50804cc5f..f94dd76d6 100755 --- a/smoketests/test_env.py +++ b/smoketests/test_env.py @@ -108,7 +108,7 @@ def auth_token(self, try_storage: bool) -> str: \b Usage: -1. run "make shell" to get a shell in the container +1. run "just shell" to get a shell in the container 2. then do "cd smoketests" 3. run "./setup_tests.py" if you are running the tests for the first time. 4. run "./test_env.py [{"|".join(ENVIRONMENTS)}]" diff --git a/systemtests/README.rst b/systemtests/README.rst index bac1aa168..01ee5a49a 100644 --- a/systemtests/README.rst +++ b/systemtests/README.rst @@ -18,7 +18,7 @@ The system tests should be run from inside the "web" container to make sure all requirements are available. They are invoked by running pytest in the ``systemtests/`` directory:: - $ make shell + $ just shell app@132812ebe909:/app$ cd systemtests/ app@132812ebe909:/app/systemtests$ pytest --target-env stage