Merge pull request #1581 from terencehonles/fix-blind-read-of-multipa…

…rt-epilogue

fix blind readline of multipart epilogue

CHANGES.rst

@@ -4,6 +4,9 @@ CHANGES
 1.3.0 (XXXX-XX-XX)
 ------------------
 
+- Multipart reader accepts multipart messages with or without their epilogue
+  to consistently handle valid and legacy behaviors #1526 #1581
+
 - Separate read + connect + request timeouts # 1523
 
 - Fix polls demo run application #1487

aiohttp/multipart.py

@@ -631,7 +631,20 @@ def _read_boundary(self):
             pass
         elif chunk == self._boundary + b'--':
             self._at_eof = True
-            yield from self._readline()
+            epilogue = yield from self._readline()
+            next_line = yield from self._readline()
+
+            # the epilogue is expected and then either the end of input or the
+            # parent multipart boundary, if the parent boundary is found then
+            # it should be marked as unread and handed to the parent for
+            # processing
+            if next_line[:2] == b'--':
+                self._unread.append(next_line)
+            # otherwise the request is likely missing an epilogue and both
+            # lines should be passed to the parent for processing
+            # (this handles the old behavior gracefully)
+            else:
+                self._unread.extend([next_line, epilogue])
         else:
             raise ValueError('Invalid boundary %r, expected %r'
                              % (chunk, self._boundary))

tests/test_py35/test_multipart_35.py

@@ -24,7 +24,12 @@ def unread_data(self, data):
 
 
 async def test_async_for_reader(loop):
-    data = [{"test": "passed"}, 42, b'plain text', b'aiohttp\n']
+    data = [
+        {"test": "passed"},
+        42,
+        b'plain text',
+        b'aiohttp\n',
+        b'no epilogue']
     reader = aiohttp.MultipartReader(
         headers={h.CONTENT_TYPE: 'multipart/mixed; boundary=":"'},
         content=Stream(b'\r\n'.join([
@@ -51,17 +56,31 @@ async def test_async_for_reader(loop):
             b'',
             b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03K\xcc\xcc\xcf())'
             b'\xe0\x02\x00\xd6\x90\xe2O\x08\x00\x00\x00',
+            b'--::',
+            b'Content-Type: multipart/related; boundary=":::"',
+            b'',
+            b'--:::',
+            b'Content-Type: text/plain',
+            b'',
+            data[4],
+            b'--:::--',
             b'--::--',
             b'',
             b'--:--',
             b''])))
     idata = iter(data)
-    async for part in reader:
-        if isinstance(part, aiohttp.BodyPartReader):
-            assert next(idata) == (await part.json())
-        else:
-            async for subpart in part:
-                assert next(idata) == await subpart.read(decode=True)
+
+    async def check(reader):
+        async for part in reader:
+            if isinstance(part, aiohttp.BodyPartReader):
+                if part.headers[h.CONTENT_TYPE] == 'application/json':
+                    assert next(idata) == (await part.json())
+                else:
+                    assert next(idata) == await part.read(decode=True)
+            else:
+                await check(part)
+
+    await check(reader)
 
 
 async def test_async_for_bodypart(loop):