Merge branch 'robust-integ-tests' into develop

* robust-integ-tests:
  Use waiter to ensure S3 key exists before proceeding

Author: jamesls

awscli/testutils.py

@@ -775,6 +775,16 @@ def head_object(self, bucket_name, key_name):
         response = client.head_object(Bucket=bucket_name, Key=key_name)
         return response
 
+    def wait_until_key_exists(self, bucket_name, key_name, extra_params=None,
+                              min_successes=3):
+        client = self.create_client_for_bucket(bucket_name)
+        waiter = client.get_waiter('object_exists')
+        params = {'Bucket': bucket_name, 'Key': key_name}
+        if extra_params is not None:
+            params.update(extra_params)
+        for _ in range(min_successes):
+            waiter.wait(**params)
+
     def assert_no_errors(self, p):
         self.assertEqual(
             p.rc, 0,

tests/integration/customizations/s3/test_plugin.py

@@ -1817,6 +1817,7 @@ def test_verify_endpoint_url_is_used(self):
 
 class TestSSERelatedParams(BaseS3IntegrationTest):
     def download_and_assert_kms_object_integrity(self, bucket, key, contents):
+        self.wait_until_key_exists(bucket, key)
         # Ensure the kms object can be download it by downloading it
         # with --sse aws:kms is enabled to ensure sigv4 is used on the
         # download, as it is required for kms.
@@ -2011,6 +2012,9 @@ def setUp(self):
 
     def download_and_assert_sse_c_object_integrity(
             self, bucket, key, encrypt_key, contents):
+        self.wait_until_key_exists(bucket, key,
+                                   {'SSECustomerKey': encrypt_key,
+                                    'SSECustomerAlgorithm': 'AES256'})
         download_filename = os.path.join(self.files.rootdir, 'tmp', key)
         p = aws('s3 cp s3://%s/%s %s --sse-c AES256 --sse-c-key %s' % (
             bucket, key, download_filename, encrypt_key))