diff --git a/.readthedocs.yml b/.readthedocs.yml index 6ccbcc4048..e03a976672 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,12 +1,15 @@ # .readthedocs.yml +version: 2 build: - image: latest - + os: "ubuntu-20.04" + tools: + python: "3.10" formats: [] python: - pip_install: true - version: 3.6 - extra_requirements: + install: + - method: pip + path: . + extra_requirements: - docs diff --git a/README.md b/README.md index faa4ae57fa..95cae5e4a9 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,8 @@ Please leave us [your feedback](https://forms.gle/i64fowQmiVhMMC7f9) on how we c ## Installation -0. We support `Linux` platform, `Python 3.6`, `3.7`, `3.8`, `3.9` and `3.10` - * **`Python 3.5` is not supported!** +0. DeepPavlov supports `Linux`, `Windows 10+` (through WSL/WSL2), `MacOS` (Big Sur+) platforms, `Python 3.6`, `3.7`, `3.8`, `3.9` and `3.10`. + Depending on the model used, you may need from 4 to 16 GB RAM. 1. Create and activate a virtual environment: * `Linux` @@ -99,6 +99,7 @@ By default, DeepPavlov installs models requirements from PyPI. PyTorch from PyPI capability. To run supported DeepPavlov models on GPU you should have [CUDA](https://developer.nvidia.com/cuda-toolkit) compatible with used GPU and [PyTorch version](deeppavlov/requirements/pytorch.txt) required by DeepPavlov models. See [docs](https://docs.deeppavlov.ai/en/master/intro/quick_start.html#using-gpu) for details. +GPU with Pascal or newer architecture and 4+ GB VRAM is recommended. ### Command line interface (CLI) diff --git a/deeppavlov/_meta.py b/deeppavlov/_meta.py index 3eb18c6604..b5a90674f9 100644 --- a/deeppavlov/_meta.py +++ b/deeppavlov/_meta.py @@ -1,4 +1,4 @@ -__version__ = '1.4.0' +__version__ = '1.5.0' __author__ = 'Neural Networks and Deep Learning lab, MIPT' __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.' __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot'] diff --git a/deeppavlov/core/commands/train.py b/deeppavlov/core/commands/train.py index b675ffab99..5062c20d20 100644 --- a/deeppavlov/core/commands/train.py +++ b/deeppavlov/core/commands/train.py @@ -92,7 +92,9 @@ def train_evaluate_model_from_config(config: Union[str, Path, dict], if iterator is None: try: data = read_data_by_config(config) - if config.get('train', {}).get('val_every_n_epochs') and not data.get('valid'): + # TODO: check class objects, not strings + is_mtl = config['dataset_reader']['class_name'] == 'multitask_reader' + if config.get('train', {}).get('val_every_n_epochs') and not data.get('valid') and not is_mtl: error_message = 'The value "val_every_n_epochs" is set in the config but no validation data is provided' raise AttributeError(error_message) except ConfigError as e: diff --git a/docs/features/models/NER.ipynb b/docs/features/models/NER.ipynb index c1c881831a..da8473f5e3 100644 --- a/docs/features/models/NER.ipynb +++ b/docs/features/models/NER.ipynb @@ -80,14 +80,14 @@ "metadata": {}, "outputs": [], "source": [ - "!python -m deeppavlov install ner_ontonotes_bert_torch" + "!python -m deeppavlov install ner_ontonotes_bert" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "`ner_ontonotes_bert_torch` is the name of the model's *config_file*. [What is a Config File?](http://docs.deeppavlov.ai/en/master/intro/configuration.html) \n", + "`ner_ontonotes_bert` is the name of the model's *config_file*. [What is a Config File?](http://docs.deeppavlov.ai/en/master/intro/configuration.html) \n", "\n", "Configuration file defines the model and describes its hyperparameters. To use another model, change the name of the *config_file* here and further.\n", "The full list of NER models with their config names can be found in the [table](#3.-Models-list).\n", @@ -126,7 +126,7 @@ "source": [ "from deeppavlov import build_model\n", "\n", - "ner_model = build_model('ner_ontonotes_bert_torch', download=True, install=True)" + "ner_model = build_model('ner_ontonotes_bert', download=True, install=True)" ] }, { @@ -180,7 +180,7 @@ "metadata": {}, "outputs": [], "source": [ - "! python -m deeppavlov interact ner_ontonotes_bert_torch -d" + "! python -m deeppavlov interact ner_ontonotes_bert -d" ] }, { @@ -198,7 +198,7 @@ "metadata": {}, "outputs": [], "source": [ - "! python -m deeppavlov predict ner_ontonotes_bert_torch -f " + "! python -m deeppavlov predict ner_ontonotes_bert -f " ] }, { @@ -224,7 +224,7 @@ "source": [ "from deeppavlov import evaluate_model\n", "\n", - "model = evaluate_model('ner_ontonotes_bert_torch', download=True)" + "model = evaluate_model('ner_ontonotes_bert', download=True)" ] }, { @@ -240,7 +240,7 @@ "metadata": {}, "outputs": [], "source": [ - "! python -m deeppavlov evaluate ner_ontonotes_bert_torch" + "! python -m deeppavlov evaluate ner_ontonotes_bert" ] }, { @@ -275,7 +275,7 @@ "from deeppavlov import train_model\n", "from deeppavlov.core.commands.utils import parse_config\n", "\n", - "model_config = parse_config('ner_ontonotes_bert_torch')\n", + "model_config = parse_config('ner_ontonotes_bert')\n", "\n", "# dataset that the model was trained on\n", "print(model_config['dataset_reader']['data_path'])" @@ -401,7 +401,7 @@ "metadata": {}, "outputs": [], "source": [ - "! python -m deeppavlov train ner_ontonotes_bert_torch" + "! python -m deeppavlov train ner_ontonotes_bert" ] }, { diff --git a/docs/features/models/multitask_bert.rst b/docs/features/models/multitask_bert.rst index 7344ecbd96..eb83618056 100644 --- a/docs/features/models/multitask_bert.rst +++ b/docs/features/models/multitask_bert.rst @@ -2,7 +2,8 @@ Multi-task BERT in DeepPavlov ============================= Multi-task BERT in DeepPavlov is an implementation of BERT training algorithm published in the paper -`Multi-Task Deep Neural Networks for Natural Language Understanding `_. +`Knowledge Transfer Between Tasks and Languages in the Multi-task +Encoder-agnostic Transformer-based Models `_. The idea is to share BERT body between several tasks. This is necessary if a model pipe has several components using BERT and the amount of GPU memory is limited. Each task has its own 'head' part attached to the diff --git a/docs/integrations/rest_api.rst b/docs/integrations/rest_api.rst index 3913662d11..0cb35e56ca 100644 --- a/docs/integrations/rest_api.rst +++ b/docs/integrations/rest_api.rst @@ -117,3 +117,26 @@ Here are POST request payload examples for some of the library models: | | |  "question_raw":["What strained the relationship between Great Britain and its colonies?"]} | +-----------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ +REST API Usage Example +====================== + +To start server with ``squad_bert`` model run: + +.. code:: bash + + python -m deeppavlov riseapi squad_bert -id + +To get response from this model on another terminal run: + +.. code:: bash + + curl -X POST http://0.0.0.0:5000/model -H 'Content-Type: application/json' -d '{ + "context_raw": [ + "All work and no play makes Jack a dull boy.", + "I used to be an adventurer like you, then I took an arrow in the knee." + ], + "question_raw": [ + "What makes Jack a dull boy?", + "Who I used to be?" + ] + }' diff --git a/docs/intro/installation.rst b/docs/intro/installation.rst index e79698e626..6c8c51a4c9 100644 --- a/docs/intro/installation.rst +++ b/docs/intro/installation.rst @@ -1,13 +1,16 @@ Installation ============ +DeepPavlov supports **Linux**, **Windows 10+** (through WSL/WSL2), **MacOS** (Big Sur+) platforms, **Python 3.6-3.10**. +Depending on the model used, you may need from 4 to 16 GB RAM. -We support ``Linux`` platform, ``Python 3.6``, ``3.7``, ``3.8``, ``3.9`` and ``3.10``. - -.. note:: - - * ``Python 3.5`` is not supported! +Install with pip +~~~~~~~~~~~~~~~~ +You should install DeepPavlov in a `virtual environment `_. If you’re +unfamiliar with Python virtual environments, take a look at this +`guide `_. A virtual +environment makes it easier to manage different projects, and avoid compatibility issues between dependencies. #. Create a virtual environment: @@ -15,29 +18,52 @@ We support ``Linux`` platform, ``Python 3.6``, ``3.7``, ``3.8``, ``3.9`` and ``3 python -m venv env -#. Activate the environment: +#. Activate the virtual environment on Linux (`source` could be replaced with `.`): - * Linux - - .. code:: bash + .. code:: bash - source ./env/bin/activate + source env/bin/activate -#. Install the package inside this virtual environment: +#. Install DeepPavlov inside this virtual environment: .. code:: bash pip install deeppavlov +Install from source +~~~~~~~~~~~~~~~~~~~ + +Install DeepPavlov **dev** branch from source with the following command: + + .. code:: bash + + pip install git+http://github.com/deeppavlov/DeepPavlov@dev + +This command installs the bleeding edge dev version rather than the latest release version. The dev version is useful +for staying up-to-date with the latest developments. For instance, if a bug has been fixed since the last release but +a new release hasn’t been rolled out yet. However, this means the dev version may not always be stable. + +Editable install +~~~~~~~~~~~~~~~~ + +You will need an editable install if you want to make changes in the DeepPavlov source code that immediately take place +without requiring a new installation. + +Clone the repository and install DeepPavlov with the following commands: + + .. code:: bash + + git clone http://github.com/deeppavlov/DeepPavlov.git + pip install -e DeepPavlov Docker Images -------------- +~~~~~~~~~~~~~ We have built several DeepPavlov based Docker images, which include: * DeepPavlov based Jupyter notebook Docker image; * Docker images which serve some of our models and allow to access them - via REST API (``riseapi`` mode). + via REST API (:doc:`riseapi ` mode). Here is our `DockerHub repository `_ with images and deployment instructions. diff --git a/docs/intro/quick_start.rst b/docs/intro/quick_start.rst index f2b6e89ec1..66626b4ffd 100644 --- a/docs/intro/quick_start.rst +++ b/docs/intro/quick_start.rst @@ -132,6 +132,7 @@ Using GPU To run or train **PyTorch**-based DeepPavlov models on GPU you should have `CUDA `__ installed on your host machine, and install model's package requirements. CUDA version should be compatible with DeepPavlov :dp_file:`required PyTorch version `. +GPU with Pascal or newer architecture and 4+ GB VRAM is recommended. .. warning:: If you use latest NVIDIA architecture, PyTorch installed from PyPI using DeepPavlov could not support your device @@ -198,17 +199,97 @@ Text Question Answering Text Question Answering component answers a question based on a given context (e.g, a paragraph of text), where the answer to the question is a segment of the context. -.. table:: - :widths: auto +.. code:: python - +----------+------------------------------------------------------------------------------------+-------------------------------------------+ - | Language | DeepPavlov config | Demo | - +==========+====================================================================================+===========================================+ - | En | :config:`squad_bert ` | https://demo.deeppavlov.ai/#/en/textqa | - +----------+------------------------------------------------------------------------------------+-------------------------------------------+ - | Ru | :config:`squad_ru_bert ` | https://demo.deeppavlov.ai/#/ru/textqa | - +----------+------------------------------------------------------------------------------------+-------------------------------------------+ + from deeppavlov import build_model + model = build_model('squad_bert', download=True, install=True) + contexts = ['DeepPavlov is a library for NLP and dialog systems.', 'All work and no play makes Jack a dull boy'] + questions = ['What is DeepPavlov?', 'What makes Jack a dull boy?'] + answer, answers_start_idx, score = model(contexts, questions) + print(answer) + +.. code:: bash + + ['a library for NLP and dialog systems', 'All work and no play'] + +To get list of available models for Text Question Answering see :doc:`documentation `. + +Open-Domain Question Answering +============================== + +Open Domain Question Answering (ODQA) answers any question based on the document collection covering a wide range of +topics. The ODQA task combines two challenges of document retrieval (finding the relevant articles) with that of machine +comprehension of text (identifying the answer span from those articles). This component can be used to answer questions +based on the company knowledge base. + +.. code:: python + + from deeppavlov import build_model + + model = build_model('en_odqa_infer_wiki', download=True, install=True) + questions = ["What is the name of Darth Vader's son?", 'Who was the first president of France?'] + answer, answer_score, answer_place = model(questions) + print(answer) + +.. code:: bash + + ['Luke Skywalker', 'Louis-Napoleon Bonaparte'] + +To get list of available models for Open-Domain Question Answering see :doc:`documentation `. + +Knowledge Base Question Answering +================================= + +Knowledge Base Question Answering (KBQA) answers any question based on Knowledge Base (Knowledge Graph) - +a comprehensive repository of information about a given domain or a number of domains that reflects the ways we model +knowledge about a given subject or subjects, in terms of concepts, entities, properties, and relationships. KBQA models +validate questions against a preconfigured list of question templates, disambiguate entities using Entity Linking, +and answer questions asked in natural language. + +.. code:: python + + from deeppavlov import build_model + + model = build_model('kbqa_cq_en', download=True, install=True) + questions = ['What is the currency of Sweden?', 'When did the Korean War end?'] + answers, answer_ids, query = model(questions) + print(answers) + +.. code:: bash + + ['Swedish krona', '27 July 1953'] + +To get list of available models for Knowledge Base Question Answering see :doc:`documentation `. + +Classification (insult and paraphrase detection, sentiment analysis, topic classification) +========================================================================================== + +Insult detection predicts whether a text (e.g, post or speech in some public discussion) is considered insulting to one +of the persons it is related to. + +Sentiment analysis is a task of classifying the polarity of the the given sequence. + +The models trained for the paraphrase detection task identify whether two sentences expressed with different words +convey the same meaning. + +Topic classification refers to the task of classifying an utterance by the topic which belongs to the conversational +domain. + +.. code:: python + + from deeppavlov import build_model + + model = build_model('insults_kaggle_bert', download=True, install=True) + phrases = ['You are kind of stupid', 'You are a wonderful person!'] + labels = model(phrases) + print(labels) + +.. code:: bash + + ['Insult', 'Not Insult'] + +To get list of available models for Classification see :doc:`documentation `. Name Entity Recognition ======================= @@ -217,47 +298,63 @@ Named Entity Recognition (NER) classifies tokens in text into predefined categor (tags), such as person names, quantity expressions, percentage expressions, names of locations, organizations, as well as expression of time, currency and others. -.. table:: - :widths: auto +.. code:: python + + from deeppavlov import build_model + + model = build_model('ner_ontonotes_bert', download=True, install=True) + phrases = ['Bob Ross lived in Florida', 'Elon Musk founded Tesla'] + tokens, tags = model(phrases) + print(tokens, tags, sep='\n') + +.. code:: bash + + [['Bob', 'Ross', 'lived', 'in', 'Florida'], ['Elon', 'Musk', 'founded', 'Tesla']] + [['B-PERSON', 'I-PERSON', 'O', 'O', 'B-GPE'], ['B-PERSON', 'I-PERSON', 'O', 'B-ORG']] + +To get list of available models for Name Entity Recognition see :doc:`documentation `. + +Entity Extraction +================= + +Entity Detection is the task of identifying entity mentions in text with corresponding entity types. +Entity Linking is the task of finding knowledge base entity ids for entity mentions in text. +Entity Extraction configs perform subsequent Entity Detection and Entity Linking of extracted entity mentions. + +.. code:: python + + from deeppavlov import build_model + + model = build_model('entity_extraction_en', download=True, install=True) + phrases = ['Forrest Gump is a comedy-drama film directed by Robert Zemeckis and written by Eric Roth.'] + entity_substr, tags, entity_offsets, entity_ids, entity_conf, entity_pages, entity_labels = model(phrases) + print(entity_substr, tags, entity_ids, entity_labels, sep='\n') - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ - | Language | DeepPavlov config | Demo | - +==========+================================================================================================+===========================================+ - | Multi | :config:`ner_ontonotes_bert_mult ` | https://demo.deeppavlov.ai/#/mu/ner | - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ - | En | :config:`ner_ontonotes_bert_mult ` | https://demo.deeppavlov.ai/#/en/ner | - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ - | Ru | :config:`ner_rus_bert ` | https://demo.deeppavlov.ai/#/ru/ner | - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ +.. code:: bash + [['forrest gump', 'robert zemeckis', 'eric roth']] + [['WORK_OF_ART', 'PERSON', 'PERSON']] + [[['Q134773', 'Q552213', 'Q12016774'], ['Q187364', 'Q36951156'], ['Q942932', 'Q89320386', 'Q89909683']]] + [[['Forrest Gump', 'Forrest Gump', 'Forrest Gump'], ['Robert Zemeckis', 'Welcome to Marwen'], ['Eric Roth', 'Eric Roth', 'Eric W Roth']]] -Insult Detection -================ +To get list of available models for Entity Extraction see :doc:`documentation `. -Insult detection predicts whether a text (e.g, post or speech in some -public discussion) is considered insulting to one of the persons it is -related to. +Spelling Correction +=================== -.. table:: - :widths: auto +Spelling Correction models detect and correct spelling errors in texts. - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ - | Language | DeepPavlov config | Demo | - +==========+================================================================================================+===========================================+ - | En | :config:`insults_kaggle_bert ` | https://demo.deeppavlov.ai/#/en/insult | - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ +.. code:: python + from deeppavlov import build_model -Paraphrase Detection -==================== + model = build_model('brillmoore_wikitypos_en', download=True, install=True) + phrases_w_typos = ['I think this is the begining of a beautifull frendship.', "I'll be bak"] + correct_phrases = model(phrases_w_typos) + print(correct_phrases) -Detect if two given texts have the same meaning. +.. code:: bash -.. table:: - :widths: auto + ['i think this is the beginning of a beautiful friendship.', "i'll be back"] - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ - | Language | DeepPavlov config | Demo | - +==========+================================================================================================+===========================================+ - | Ru | :config:`paraphraser_rubert ` | None | - +----------+------------------------------------------------------------------------------------------------+-------------------------------------------+ +To get list of available models for Spelling Correction see :doc:`documentation `.