Merge branch 'pytorch:main' into main

.github/workflows/ci_cpu.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Lint with black
         run: black --check --diff --color .
       - name: Check import order with isort
-        run: isort -v -l 88 -o opacus --lines-after-imports 2 -m 3 --trailing-comma --check-only .
+        run: isort -l 88 -o opacus --lines-after-imports 2 -m 3 --trailing-comma --check-only .
 
   ########### UNIT TESTS ##############
   unittest_py38_torch_release:
@@ -52,13 +52,21 @@ jobs:
       - name: Run unit tests
         run: |
           mkdir unittest-py38-release-reports
-          coverage run -m pytest --doctest-modules -p conftest --junitxml=unittest-py38-release-reports/junit.xml opacus
+          coverage run -m pytest --doctest-modules -p conftest opacus
           coverage report -i -m
+          # Format into xml to be used for coveralls
+          coverage xml -i
       - name: Store test results
         uses: actions/upload-artifact@v4
         with:
           name: unittest-py38-release-reports
           path: unittest-py38-release-reports
+      - name: Send coverage to Coveralls (parallel)
+        uses: coverallsapp/github-action@v2
+        with:
+          format: cobertura
+          parallel: true
+          flag-name: run-1
 
   unittest_py39_torch_release:
     runs-on: ubuntu-latest
@@ -77,13 +85,19 @@ jobs:
       - name: Run unit tests
         run: |
           mkdir unittest-py39-release-reports
-          coverage run -m pytest --doctest-modules -p conftest --junitxml=unittest-py39-release-reports/junit.xml opacus
-          coverage report -i -m
+          coverage run -m pytest --doctest-modules -p conftest opacus
+          coverage xml -i
       - name: Store test results
         uses: actions/upload-artifact@v4
         with:
           name: unittest-py39-release-reports
           path: unittest-py39-release-reports
+      - name: Send coverage to Coveralls (parallel)
+        uses: coverallsapp/github-action@v2
+        with:
+          format: cobertura
+          parallel: true
+          flag-name: run-2
 
   prv_accountant_values:
     runs-on: ubuntu-latest
@@ -150,11 +164,18 @@ jobs:
           coverage run examples/mnist.py --lr 0.25 --sigma 0.7 -c 1.5 --batch-size 64 --epochs 1 --data-root runs/mnist/data --n-runs 1 --device cpu
           python -c "import torch; accuracy = torch.load('run_results_mnist_0.25_0.7_1.5_64_1.pt'); exit(0) if (accuracy[0]>0.78 and accuracy[0]<0.95) else exit(1)"
           coverage report -i -m
+          coverage xml -i
       - name: Store test results
         uses: actions/upload-artifact@v4
         with:
           name: mnist-cpu-reports
           path: runs/mnist/test-reports
+      - name: Send coverage to Coveralls (parallel)
+        uses: coverallsapp/github-action@v2
+        with:
+          format: cobertura
+          parallel: true
+          flag-name: run-3
 
   ######## FINISH COVERALLS ##########
   finish_coveralls_parallel:
@@ -168,3 +189,4 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           parallel-finished: true
+          carryforward: "run-1,run-2,run-3"

CONTRIBUTING.md

@@ -31,11 +31,16 @@ for advanced usage).
 
 Opacus also uses [isort](https://github.com/timothycrosley/isort) to sort imports
 alphabetically and separate into sections. isort is installed easily via
-pip using `pip install isort`, and run locally by calling
+pip using `pip install isort --upgrade`, and run locally by calling
 ```bash
-isort -v -l 88 -o opacus --lines-after-imports 2 -m 3 --trailing-comma  .
+isort -l 88 -o opacus --lines-after-imports 2 -m 3 --trailing-comma  .
 ```
 from the repository root. Configuration for isort is located in .isort.cfg.
+If using `isort` versions `<5.0.0` call
+```bash
+isort -l 88 -o opacus --lines-after-imports 2 -m 3 --trailing-comma --recursive
+```
+
 
 We feel strongly that having a consistent code style is extremely important, so
 CircleCI will fail on your PR if it does not adhere to the black or flake8 formatting style or isort import ordering.
@@ -96,7 +101,7 @@ Run following command from `website` folder. It will build the docs and serve th
 ```
 
 You can also perform spell checks on documentation automatically (besides IDEs) using [```sphinxcontrib-spelling```](https://sphinxcontrib-spelling.readthedocs.io/en/latest/install.html)
-Note that you will also need [```PyEnchant```](https://pyenchant.github.io/pyenchant/) to run ```sphinxcontrib-spelling```, and thus the Enchant C library. Use this guide for ```PyEnchant```. 
+Note that you will also need [```PyEnchant```](https://pyenchant.github.io/pyenchant/) to run ```sphinxcontrib-spelling```, and thus the Enchant C library. Use this guide for ```PyEnchant```.
 
 Steps:
 1. Install the extension with pip: ```pip install sphinxcontrib-spelling```

README.md

@@ -2,6 +2,7 @@
 
 <hr/>
 
+[![PyPI Downloads](https://static.pepy.tech/badge/opacus)](https://pepy.tech/projects/opacus)
 [![GitHub Actions](https://github.com/pytorch/opacus/actions/workflows/ci_cpu.yml/badge.svg)](https://github.com/pytorch/opacus/actions/workflows/ci_cpu.yml)
 [![Coverage Status](https://coveralls.io/repos/github/pytorch/opacus/badge.svg?branch=main)](https://coveralls.io/github/pytorch/opacus?branch=main)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md)
@@ -22,6 +23,13 @@ This code release is aimed at two target audiences:
 2. Differential Privacy researchers will find this easy to experiment and tinker
    with, allowing them to focus on what matters.
 
+
+## Latest updates
+
+2024-12-18: We updated this [tutorial](https://github.com/pytorch/opacus/blob/main/tutorials/building_text_classifier.ipynb) to show how [LoRA](https://arxiv.org/abs/2106.09685) and [peft](https://huggingface.co/docs/peft/en/index) library could be used in conjuncture with DP-SGD.
+
+2024-08-20: We introduced [Fast Gradient Clipping](https://arxiv.org/abs/2009.03106) and Ghost Clipping(https://arxiv.org/abs/2110.05679) to Opacus, significantly reducing the memory requirements of DP-SGD. Please refer to our [blogpost](https://pytorch.org/blog/clipping-in-opacus/) for more information.
+
 ## Installation
 
 The latest release of Opacus can be installed via `pip`:
@@ -75,23 +83,16 @@ shows an end-to-end run using Opacus. The
 [examples](https://github.com/pytorch/opacus/tree/main/examples/) folder
 contains more such examples.
 
-### Migrating to 1.0
-
-Opacus 1.0 introduced many improvements to the library, but also some breaking
-changes. If you've been using Opacus 0.x and want to update to the latest
-release, please use this
-[Migration Guide](https://github.com/pytorch/opacus/blob/main/Migration_Guide.md)
-
 ## Learn more
 
 ### Interactive tutorials
 
 We've built a series of IPython-based tutorials as a gentle introduction to
 training models with privacy and using various Opacus features.
 
+- [Building text classifier with Differential Privacy on BERT](https://github.com/pytorch/opacus/blob/main/tutorials/building_text_classifier.ipynb)
 - [Building an Image Classifier with Differential Privacy](https://github.com/pytorch/opacus/blob/main/tutorials/building_image_classifier.ipynb)
 - [Training a differentially private LSTM model for name classification](https://github.com/pytorch/opacus/blob/main/tutorials/building_lstm_name_classifier.ipynb)
-- [Building text classifier with Differential Privacy on BERT](https://github.com/pytorch/opacus/blob/main/tutorials/building_text_classifier.ipynb)
 - [Opacus Guide: Introduction to advanced features](https://github.com/pytorch/opacus/blob/main/tutorials/intro_to_advanced_features.ipynb)
 - [Opacus Guide: Grad samplers](https://github.com/pytorch/opacus/blob/main/tutorials/guide_to_grad_sampler.ipynb)
 - [Opacus Guide: Module Validator and Fixer](https://github.com/pytorch/opacus/blob/main/tutorials/guide_to_module_validator.ipynb)
@@ -118,12 +119,12 @@ Consider citing the report if you use Opacus in your papers, as follows:
 If you want to learn more about DP-SGD and related topics, check out our series
 of blogposts and talks:
 
+- [Enabling Fast Gradient Clipping and Ghost Clipping in Opacus](https://pytorch.org/blog/clipping-in-opacus/)
 - [Differential Privacy Series Part 1 | DP-SGD Algorithm Explained](https://medium.com/pytorch/differential-privacy-series-part-1-dp-sgd-algorithm-explained-12512c3959a3)
 - [Differential Privacy Series Part 2 | Efficient Per-Sample Gradient Computation in Opacus](https://medium.com/pytorch/differential-privacy-series-part-2-efficient-per-sample-gradient-computation-in-opacus-5bf4031d9e22)
 - [PriCon 2020 Tutorial: Differentially Private Model Training with Opacus](https://www.youtube.com/watch?v=MWPwofiQMdE&list=PLUNOsx6Az_ZGKQd_p4StdZRFQkCBwnaY6&index=52)
 - [Differential Privacy on PyTorch | PyTorch Developer Day 2020](https://www.youtube.com/watch?v=l6fbl2CBnq0)
 - [Opacus v1.0 Highlights | PyTorch Developer Day 2021](https://www.youtube.com/watch?v=U1mszp8lzUI)
-- [Enabling Fast Gradient Clipping and Ghost Clipping in Opacus](https://pytorch.org/blog/clipping-in-opacus/)
 
 ## FAQ
 

dev_requirements.txt

@@ -8,7 +8,7 @@ flake8
 sphinx
 sphinx-autodoc-typehints
 mypy>=0.760
-isort
+isort>=5.0.0
 hypothesis
 tensorboard
 datasets

docs/faq.md

@@ -13,8 +13,8 @@ Yes! Opacus is open-source for public use, and it is licensed under the [Apache
 
 ## How can I report a bug or ask a question?
 
-You can report bugs by submitting GitHub issues. To submit a GitHub issue, please [click here](https://github.com/pytorch/opacus/issues).
-You can ask questions in our dedicated PyTorch [Discussion Forum](https://discuss.pytorch.org/c/opacus/29). We actively monitor questions in the PyTorch forums with the category `Opacus`.
+You can report bugs or ask questions by submitting GitHub issues. To submit a GitHub issue, please [click here](https://github.com/pytorch/opacus/issues).
+<!-- You can ask questions in our dedicated PyTorch [Discussion Forum](https://discuss.pytorch.org/c/opacus/29). We actively monitor questions in the PyTorch forums with the category `Opacus`. -->
 
 ## I'd like to contribute to Opacus. How can I do that?
 
@@ -76,7 +76,7 @@ If these interventions don’t help (or the model starts to converge but its pri
 
 ## How to deal with out-of-memory errors?
 
-Dealing with per-sample gradients will inevitably put more pressure on your memory: after all, if you want to train with batch size 64, you are looking to keep 64 copies of your parameter gradients. The first sanity check to do is to make sure that you don’t go out of memory with "standard" training (without DP). That should guarantee that you can train with batch size of 1 at least. Then, you can check your memory usage with e.g. `nvidia-smi` as usual, gradually increasing the batch size until you find your sweet spot. Note that this may mean that you still train with small batch size, which comes with its own training behavior (i.e. higher variance between batches). Training with larger batch sizes can be beneficial, and we built `virtual_step` to make this possible while still memory efficient (see *what is virtual batch size* in these FAQs).
+Dealing with per-sample gradients will inevitably put more pressure on your memory: after all, if you want to train with batch size 64, you are looking to keep 64 copies of your parameter gradients. The first sanity check to do is to make sure that you don’t go out of memory with "standard" training (without DP). That should guarantee that you can train with batch size of 1 at least. Then, you can check your memory usage with e.g. `nvidia-smi` as usual, gradually increasing the batch size until you find your sweet spot. Note that this may mean that you still train with small batch size, which comes with its own training behavior (i.e. higher variance between batches). Training with larger batch sizes can be beneficial. To this end, we built [Fast Gradient Clipping](https://pytorch.org/blog/clipping-in-opacus/) and `virtual_step` (see *what is virtual batch size* in these FAQs) to make DP-SGD memory efficient.
 
 ## What does epsilon=1.1 really mean? How about delta?
 

opacus/optimizers/adaclipoptimizer.py

@@ -53,6 +53,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         super().__init__(
             optimizer,

opacus/optimizers/ddp_perlayeroptimizer.py

@@ -48,6 +48,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         self.rank = torch.distributed.get_rank()
         self.world_size = torch.distributed.get_world_size()
@@ -79,6 +80,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         self.rank = torch.distributed.get_rank()
         self.world_size = torch.distributed.get_world_size()

opacus/optimizers/ddpoptimizer.py

@@ -38,6 +38,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         super().__init__(
             optimizer,
@@ -47,6 +48,7 @@ def __init__(
             loss_reduction=loss_reduction,
             generator=generator,
             secure_mode=secure_mode,
+            **kwargs,
         )
         self.rank = torch.distributed.get_rank()
         self.world_size = torch.distributed.get_world_size()

opacus/optimizers/ddpoptimizer_fast_gradient_clipping.py

@@ -38,6 +38,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         super().__init__(
             optimizer,
@@ -47,6 +48,7 @@ def __init__(
             loss_reduction=loss_reduction,
             generator=generator,
             secure_mode=secure_mode,
+            **kwargs,
         )
         self.rank = torch.distributed.get_rank()
         self.world_size = torch.distributed.get_world_size()

opacus/optimizers/optimizer.py

@@ -205,6 +205,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         """
 

opacus/optimizers/optimizer_fast_gradient_clipping.py

@@ -63,6 +63,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         """
 
@@ -91,6 +92,7 @@ def __init__(
             loss_reduction=loss_reduction,
             generator=generator,
             secure_mode=secure_mode,
+            **kwargs,
         )
 
     @property

opacus/optimizers/perlayeroptimizer.py

@@ -39,6 +39,7 @@ def __init__(
         loss_reduction: str = "mean",
         generator=None,
         secure_mode: bool = False,
+        **kwargs,
     ):
         assert len(max_grad_norm) == len(params(optimizer))
         self.max_grad_norms = max_grad_norm
@@ -51,6 +52,7 @@ def __init__(
             loss_reduction=loss_reduction,
             generator=generator,
             secure_mode=secure_mode,
+            **kwargs,
         )
 
     def clip_and_accumulate(self):

opacus/privacy_engine.py

@@ -136,6 +136,7 @@ def _prepare_optimizer(
             loss_reduction=loss_reduction,
             generator=generator,
             secure_mode=self.secure_mode,
+            **kwargs,
         )
 
     def _prepare_data_loader(