Implement s3 get_object_cdn_url using pre-signed urls

CHANGES.rst

@@ -422,6 +422,18 @@ Storage
   (GITHUB-1410)
   [Clemens Wolff - @c-w]
 
+- [AWS S3] Implement ``get_object_cdn_url`` for the AWS storage driver.
+
+  The AWS storage driver can now be used to generate temporary URLs that
+  grant clients read access to objects. The URLs expire after a certain
+  period of time, either configured via the ``ex_expiry`` argument or the
+  ``LIBCLOUD_S3_STORAGE_CDN_URL_EXPIRY_HOURS`` environment variable
+  (default: 24 hours).
+
+  Reported by @rvolykh.
+  (GITHUB-1403)
+  [Aaron Virshup - @avirshup]
+
 DNS
 ~~~
 

libcloud/common/aws.py

@@ -340,6 +340,8 @@ def _get_canonical_headers(self, headers):
                           for k, v in sorted(headers.items())]) + '\n'
 
     def _get_payload_hash(self, method, data=None):
+        if data is UnsignedPayloadSentinel:
+            return UNSIGNED_PAYLOAD
         if method in ('POST', 'PUT'):
             if data:
                 if hasattr(data, 'next') or hasattr(data, '__next__'):
@@ -368,6 +370,10 @@ def _get_canonical_request(self, params, headers, method, path, data):
         ])
 
 
+class UnsignedPayloadSentinel:
+    pass
+
+
 class SignedAWSConnection(AWSTokenConnection):
     version = None  # type: Optional[str]
 

libcloud/storage/drivers/s3.py

@@ -17,6 +17,8 @@
 import hmac
 import time
 from hashlib import sha1
+import os
+from datetime import datetime
 
 import libcloud.utils.py3
 
@@ -32,13 +34,14 @@
 from libcloud.utils.py3 import urlquote
 from libcloud.utils.py3 import b
 from libcloud.utils.py3 import tostring
+from libcloud.utils.py3 import urlencode
 
 from libcloud.utils.xml import fixxpath, findtext
 from libcloud.utils.files import read_in_chunks
 from libcloud.common.types import InvalidCredsError, LibcloudError
 from libcloud.common.base import ConnectionUserAndKey, RawResponse
 from libcloud.common.aws import AWSBaseResponse, AWSDriver, \
-    AWSTokenConnection, SignedAWSConnection
+    AWSTokenConnection, SignedAWSConnection, UnsignedPayloadSentinel
 
 from libcloud.storage.base import Object, Container, StorageDriver
 from libcloud.storage.types import ContainerError
@@ -108,6 +111,12 @@
 # ex_iterate_multipart_uploads.
 RESPONSES_PER_REQUEST = 100
 
+S3_CDN_URL_DATETIME_FORMAT = '%Y%m%dT%H%M%SZ'
+S3_CDN_URL_DATE_FORMAT = '%Y%m%d'
+S3_CDN_URL_EXPIRY_HOURS = float(
+    os.getenv('LIBCLOUD_S3_CDN_URL_EXPIRY_HOURS', '24')
+)
+
 
 class S3Response(AWSBaseResponse):
     namespace = None
@@ -1112,6 +1121,67 @@ def __init__(self, key, secret=None, secure=True, host=None, port=None,
     def list_regions(self):
         return REGION_TO_HOST_MAP.keys()
 
+    def get_object_cdn_url(self, obj,
+                           ex_expiry=S3_CDN_URL_EXPIRY_HOURS):
+        """
+        Return a "presigned URL" for read-only access to object
+
+        AWS only - requires AWS signature V4 authenticaiton.
+
+        :param obj: Object instance.
+        :type  obj: :class:`Object`
+
+        :param ex_expiry: The number of hours after which the URL expires.
+                          Defaults to 24 hours or the value of the environment
+                          variable "LIBCLOUD_S3_STORAGE_CDN_URL_EXPIRY_HOURS",
+                          if set.
+        :type  ex_expiry: ``float``
+
+        :return: Presigned URL for the object.
+        :rtype: ``str``
+        """
+
+        # assemble data for the request we want to pre-sign
+        # see: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html # noqa
+        object_path = self._get_object_path(obj.container, obj.name)
+        now = datetime.utcnow()
+        duration_seconds = int(ex_expiry * 3600)
+        credparts = (
+            self.key,
+            now.strftime(S3_CDN_URL_DATE_FORMAT),
+            self.region,
+            's3',
+            'aws4_request')
+        params_to_sign = {
+            'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
+            'X-Amz-Credential': '/'.join(credparts),
+            'X-Amz-Date': now.strftime(S3_CDN_URL_DATETIME_FORMAT),
+            'X-Amz-Expires': duration_seconds,
+            'X-Amz-SignedHeaders': 'host'}
+        headers_to_sign = {'host': self.connection.host}
+
+        # generate signature for the pre-signed request
+        signature = self.connection.signer._get_signature(
+            params=params_to_sign,
+            headers=headers_to_sign,
+            dt=now,
+            method='GET',
+            path=object_path,
+            data=UnsignedPayloadSentinel
+        )
+
+        # Create final params for pre-signed URL
+        params = params_to_sign.copy()
+        params['X-Amz-Signature'] = signature
+
+        return '{scheme}://{host}:{port}{path}?{params}'.format(
+            scheme='https' if self.secure else 'http',
+            host=self.connection.host,
+            port=self.connection.port,
+            path=object_path,
+            params=urlencode(params),
+        )
+
 
 class S3USEast2Connection(S3SignatureV4Connection):
     host = S3_US_EAST2_HOST

libcloud/test/storage/test_aurora.py

@@ -16,6 +16,7 @@
 import sys
 import unittest
 
+from libcloud.common.types import LibcloudError
 from libcloud.storage.drivers.auroraobjects import AuroraObjectsStorageDriver
 from libcloud.test.storage.test_s3 import S3MockHttp, S3Tests
 
@@ -30,6 +31,15 @@ def setUp(self):
         S3MockHttp.type = None
         self.driver = self.create_driver()
 
+    def test_get_object_cdn_url(self):
+        self.mock_response_klass.type = 'get_object'
+        obj = self.driver.get_object(container_name='test2',
+                                     object_name='test')
+
+        with self.assertRaises(LibcloudError):
+            self.driver.get_object_cdn_url(obj)
+
+
 
 if __name__ == '__main__':
     sys.exit(unittest.main())

libcloud/test/storage/test_s3.py

@@ -43,7 +43,7 @@
 from libcloud.storage.types import InvalidContainerNameError
 from libcloud.storage.types import ObjectDoesNotExistError
 from libcloud.storage.types import ObjectHashMismatchError
-from libcloud.storage.drivers.s3 import BaseS3Connection
+from libcloud.storage.drivers.s3 import BaseS3Connection, S3SignatureV4Connection
 from libcloud.storage.drivers.s3 import S3StorageDriver, S3USWestStorageDriver
 from libcloud.storage.drivers.s3 import CHUNK_SIZE
 from libcloud.utils.py3 import b
@@ -528,6 +528,24 @@ def test_get_container_success(self):
         container = self.driver.get_container(container_name='test1')
         self.assertTrue(container.name, 'test1')
 
+    def test_get_object_cdn_url(self):
+        self.mock_response_klass.type = 'get_object'
+        obj = self.driver.get_object(container_name='test2',
+                                     object_name='test')
+
+        # cdn urls can only be generated using a V4 connection
+        if issubclass(self.driver.connectionCls, S3SignatureV4Connection):
+            cdn_url = self.driver.get_object_cdn_url(obj, ex_expiry=12)
+            url = urlparse.urlparse(cdn_url)
+            query = urlparse.parse_qs(url.query)
+
+            self.assertEqual(len(query['X-Amz-Signature']), 1)
+            self.assertGreater(len(query['X-Amz-Signature'][0]), 0)
+            self.assertEqual(query['X-Amz-Expires'], ['43200'])
+        else:
+            with self.assertRaises(NotImplementedError):
+                self.driver.get_object_cdn_url(obj)
+
     def test_get_object_container_doesnt_exist(self):
         # This method makes two requests which makes mocking the response a bit
         # trickier