TST/CI: refactor test suite to use TeenyTweetyNet, fix #330

generating test data:
- rename tweetynet configs
- add teenytweetynet configs
- modify test_data_generate.py script
  - so it finds all configs
  - have it make model subdirectories in results/
- rewrite Makefile
  - differentiate 'all' and 'ci' generated test data
  - change commands, variables, urls to data

refactoring tests:
- add `model` parameters to fixtures:
  - to `specific_config` fixture
  - to `specific_config_toml` fixture
  - to `dataframe` fixtures
  - change `previous_run_path` fixture to `previous_run_path_factory`
    - use `model` as argument to `_previous_run_path` function
      returned by factory
- add `models` command-line option for pytest
  + that will parametrize any test that specifies `models` fixture
    with whatever arguments are passed in at command line
- use `model` fixture in tests
  - add model fixture to unit tests in test_core/ and test_cli/
  - use `model` fixture in a test in test_labeled_timebins
  - use `previous_run_path_factory` with `model` in
    test_cli/test_learncurve.py

CI:
- have ci.yml download just test data generated for ci
- have ci run pytest using command-line option models,
  and specifying only teenytweetynet for now
  (default but make it explicit anyway)
- fix the `fix_prep_csv_paths` script so that it correctly
  finds prep csv files with the new test data directory
  names and structure

other refactoring
- move src/scripts to tests/scripts
- rename test_data to data_for_tests
  + so it doesn't look like a sub-package of tests to `pytest`

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
       - name: run tests
         run: |
           make test-data-download-source
-          make test-data-download-generate
+          make test-data-download-generated-ci
           poetry install
-          poetry run python ./src/scripts/test_data/fix_prep_csv_paths.py
-          poetry run pytest
+          poetry run python ./tests/scripts/fix_prep_csv_paths.py
+          poetry run pytest --models teenytweetynet

.gitignore

@@ -18,6 +18,6 @@ build/
 doc/_build/
 
 # test data
-tests/test_data/source/
-tests/test_data/generated/
+tests/data_for_tests/source/
+tests/data_for_tests/generated/
 *.tar.gz

Makefile

@@ -1,33 +1,57 @@
-SOURCE_TEST_DATA_TAR=tests/test_data/source/source_test_data.tar.gz
-SOURCE_TEST_DATA_URL=https://osf.io/7ru4s/download
+TEST_DATA_GENERATE_SCRIPT=./tests/scripts/generate_data_for_tests.py
 
-TEST_DATA_GENERATE_SCRIPT=./src/scripts/test_data/test_data_generate.py
-GENERATED_TEST_DATA_TAR=tests/test_data/generated/generated_test_data.tar.gz
-GENERATED_TEST_DATA_URL=https://osf.io/q76xd/download
-GENERATED_TEST_DATA_TOP_LEVEL_DIRS=tests/test_data/generated/configs tests/test_data/generated/prep tests/test_data/generated/results
+DATA_FOR_TESTS_DIR=./tests/data_for_tests/
+GENERATED_TEST_DATA_DIR=${DATA_FOR_TESTS_DIR}generated/
+CONFIGS_DIR=${GENERATED_TEST_DATA_DIR}configs
+PREP_DIR=${GENERATED_TEST_DATA_DIR}prep/
+RESULTS_DIR=${GENERATED_TEST_DATA_DIR}results/
+RESULTS_CI=$(shell ls -d ${RESULTS_DIR}*/*/teenytweetynet)
+GENERATED_TEST_DATA_CI_DIRS=${CONFIGS_DIR} ${PREP_DIR} ${RESULTS_CI}
+GENERATED_TEST_DATA_ALL_DIRS=${GENERATED_TEST_DATA_CI_DIRS} $(shell ls -d ${RESULTS_DIR}/*/*/tweetynet)
+
+SOURCE_TEST_DATA_TAR=${DATA_FOR_TESTS_DIR}source/source_test_data.tar.gz
+GENERATED_TEST_DATA_CI_TAR=${GENERATED_TEST_DATA_DIR}generated_test_data.ci.tar.gz
+GENERATED_TEST_DATA_ALL_TAR=${GENERATED_TEST_DATA_DIR}generated_test_data.tar.gz
+
+SOURCE_TEST_DATA_URL=https://osf.io/s85vh/download
+GENERATED_TEST_DATA_ALL_URL=https://osf.io/gt5xw/download
+GENERATED_TEST_DATA_CI_URL=https://osf.io/u64nt/download
 
 help:
 	@echo 'Makefile for vak                                                           			'
 	@echo '                                                                           			'
 	@echo 'Usage:                                                                     			'
-	@echo '     make test-data-clean-source          remove source test data                        '
-	@echo '     make test-data-download-source       download source test data                      '
-	@echo '     make test-data-generate              generate vak files used by tests from source data   '
-	@echo '     make test-data-clean-generate        remove generated test data          					'
-	@echo '     make test-data-tar-generate          place generated test data in compressed tar file       	'
-	@echo '     make test-data-download-generate     download generated test data .tar and expand        	'
-	@echo '     make variables              show variables defined for Makefile 					'
+	@echo '     make test-data-clean-source				remove source test data                        '
+	@echo '     make test-data-download-source          download source test data                      '
+	@echo '     make test-data-generate                 generate vak files used by tests from source data   '
+	@echo '     make test-data-clean-generated          remove generated test data          					'
+	@echo '     make test-data-tar-generated-all        place all generated test data in compressed tar file       	'
+	@echo '     make test-data-tar-generated-ci         place generated test data for CI in compressed tar file       	'
+	@echo '     make test-data-download-generated-all   download .tar with all generated test data and expand        	'
+	@echo '     make test-data-download-generated-ci    download .tar with generated test data for CI and expand        	'
+	@echo '     make variables                          show variables defined for Makefile 					'
 
 variables:
-	@echo '     SOURCE_TEST_DATA_TAR                : $(GENERATED_TEST_DATA_TAR)				'
-	@echo '     SOURCE_TEST_DATA_URL                : $(GENERATED_TEST_DATA_URL)				'
-	@echo '     TESTS_DATA_GENERATE_SCRIPT    		: $(TEST_DATA_GENERATE_SCRIPT)				'
-	@echo '     GENERATED_TEST_DATA_TAR      		: $(GENERATED_TEST_DATA_TAR)				'
-	@echo '     GENERATED_TEST_DATA_URL      		: $(GENERATED_TEST_DATA_URL)				'
-	@echo '     GENERATED_TEST_DATA_TOP_LEVEL_DIRS	: $(GENERATED_TEST_DATA_TOP_LEVEL_DIRS)		'
+	@echo '     TESTS_DATA_GENERATE_SCRIPT      : $(TEST_DATA_GENERATE_SCRIPT)				'
+	@echo ''
+	@echo '     DATA_FOR_TESTS_DIR              : $(DATA_FOR_TESTS_DIR)		'
+	@echo '     GENERATED_TEST_DATA_DIR         : $(GENERATED_TEST_DATA_DIR)		'
+	@echo '     PREP_DIR                        : $(PREP_DIR)		'
+	@echo '     RESULTS_DIR                     : $(RESULTS_DIR)		'
+	@echo '     RESULTS_CI                      : $(RESULTS_CI)		'
+	@echo '     GENERATED_TEST_DATA_CI_DIRS     : $(GENERATED_TEST_DATA_CI_DIRS)		'
+	@echo '     GENERATED_TEST_DATA_ALL_DIRS    : $(GENERATED_TEST_DATA_ALL_DIRS)		'
+	@echo ''
+	@echo '     SOURCE_TEST_DATA_TAR            : $(SOURCE_TEST_DATA_TAR)				'
+	@echo '     GENERATED_TEST_DATA_CI_TAR      : $(GENERATED_TEST_DATA_CI_TAR)				'
+	@echo '     GENERATED_TEST_DATA_ALL_TAR     : $(GENERATED_TEST_DATA_ALL_TAR)				'
+	@echo ''
+	@echo '     SOURCE_TEST_DATA_URL            : $(SOURCE_TEST_DATA_URL)				'
+	@echo '     GENERATED_TEST_DATA_ALL_URL 	: $(GENERATED_TEST_DATA_ALL_URL)				'
+	@echo '     GENERATED_TEST_DATA_CI_URL      : $(GENERATED_TEST_DATA_CI_URL)				'
 
 test-data-clean-source:
-	rm -rfv ./tests/test_data/source/*
+	rm -rfv ./tests/data_for_tests/source/*
 
 test-data-download-source:
 	wget -q $(SOURCE_TEST_DATA_URL) -O $(SOURCE_TEST_DATA_TAR)
@@ -36,14 +60,25 @@ test-data-download-source:
 test-data-generate : $(TEST_DATA_GENERATE_SCRIPT)
 	poetry run python $(TEST_DATA_GENERATE_SCRIPT)
 
-test-data-clean-generate :
-	rm -rfv ./tests/test_data/generated/*
+test-data-clean-generated :
+	rm -rfv ./tests/data_for_tests/generated/*
+
+test-data-tar-generated-all:
+	tar -czvf $(GENERATED_TEST_DATA_ALL_TAR) $(GENERATED_TEST_DATA_ALL_DIRS)
+
+test-data-tar-generated-ci:
+	tar -czvf $(GENERATED_TEST_DATA_CI_TAR) $(GENERATED_TEST_DATA_CI_DIRS)
 
-test-data-tar-generate:
-	tar -czvf $(GENERATED_TEST_DATA_TAR) $(GENERATED_TEST_DATA_TOP_LEVEL_DIRS)
+test-data-download-generated-all:
+	wget -q $(GENERATED_TEST_DATA_ALL_URL) -O $(GENERATED_TEST_DATA_ALL_TAR)
+	tar -xzf $(GENERATED_TEST_DATA_ALL_TAR)
 
-test-data-download-generate:
-	wget -q $(GENERATED_TEST_DATA_URL) -O $(GENERATED_TEST_DATA_TAR)
-	tar -xzf $(GENERATED_TEST_DATA_TAR)
+test-data-download-generated-ci:
+	wget -q $(GENERATED_TEST_DATA_CI_URL) -O $(GENERATED_TEST_DATA_CI_TAR)
+	tar -xzf $(GENERATED_TEST_DATA_CI_TAR)
 
-.PHONY: help variables test-data-clean-source test-data-download-source test-data-generate test-data-clean-generate test-data-tar-generate test-data-download-generate
+.PHONY: help variables \
+        test-data-clean-source test-data-download-source \
+        test-data-generate test-data-clean-generated \
+        test-data-tar-generated-all test-data-tar-generated-all \
+        test-data-download-generated-all test-data-download-generated-ci