Create a session/connect per thread for s3boto3

Documentation for boto3 recommends a session per thread - https://boto3.readthedocs.io/en/latest/guide/resources.html#multithreading-multiprocessing

As the storage class is (usually) only instantiated once per process, we need to set a thread local for each thread/connection used.

Fixes jschneier#268

storages/backends/s3boto3.py

@@ -1,6 +1,7 @@
 import mimetypes
 import os
 import posixpath
+import threading
 from gzip import GzipFile
 from tempfile import SpooledTemporaryFile
 
@@ -236,7 +237,7 @@ def __init__(self, acl=None, bucket=None, **settings):
 
         self._entries = {}
         self._bucket = None
-        self._connection = None
+        self._connections = threading.local()
 
         self.security_token = None
         if not self.access_key and not self.secret_key:
@@ -253,9 +254,10 @@ def connection(self):
         # Note that proxies are handled by environment variables that the underlying
         # urllib/requests libraries read. See https://github.com/boto/boto3/issues/338
         # and http://docs.python-requests.org/en/latest/user/advanced/#proxies
-        if self._connection is None:
+        connection = getattr(self._connections, 'connection', None)
+        if connection is None:
             session = boto3.session.Session()
-            self._connection = session.resource(
+            self._connections.connection = session.resource(
                 's3',
                 aws_access_key_id=self.access_key,
                 aws_secret_access_key=self.secret_key,
@@ -265,7 +267,7 @@ def connection(self):
                 endpoint_url=self.endpoint_url,
                 config=self.config
             )
-        return self._connection
+        return self._connections.connection
 
     @property
     def bucket(self):

tests/test_s3boto3.py

@@ -2,7 +2,9 @@
 from __future__ import unicode_literals
 
 import gzip
+import threading
 from datetime import datetime
+from unittest import skipIf
 
 from botocore.exceptions import ClientError
 from django.conf import settings
@@ -22,7 +24,7 @@
 class S3Boto3TestCase(TestCase):
     def setUp(self):
         self.storage = s3boto3.S3Boto3Storage()
-        self.storage._connection = mock.MagicMock()
+        self.storage._connections.connection = mock.MagicMock()
 
 
 class S3Boto3StorageTests(S3Boto3TestCase):
@@ -174,8 +176,8 @@ def test_storage_open_write(self):
     def test_auto_creating_bucket(self):
         self.storage.auto_create_bucket = True
         Bucket = mock.MagicMock()
-        self.storage._connection.Bucket.return_value = Bucket
-        self.storage._connection.meta.client.meta.region_name = 'sa-east-1'
+        self.storage._connections.connection.Bucket.return_value = Bucket
+        self.storage._connections.connection.meta.client.meta.region_name = 'sa-east-1'
 
         Bucket.meta.client.head_bucket.side_effect = ClientError({'Error': {},
                                                                   'ResponseMetadata': {'HTTPStatusCode': 404}},
@@ -342,3 +344,18 @@ def test_strip_signing_parameters(self):
             '%s?X-Amz-Date=12345678&X-Amz-Signature=Signature' % expected), expected)
         self.assertEqual(self.storage._strip_signing_parameters(
             '%s?expires=12345678&signature=Signature' % expected), expected)
+
+    @skipIf(threading is None, 'Test requires threading')
+    def test_connection_threading(self):
+        connections = []
+
+        def thread_storage_connection():
+            connections.append(self.storage.connection)
+
+        for x in range(2):
+            t = threading.Thread(target=thread_storage_connection)
+            t.start()
+            t.join()
+
+        # Connection for each thread needs to be unique
+        self.assertIsNot(connections[0], connections[1])