Environment
- python 3.7
Error
- An error occurred (404) when calling the HeadObject operation
Situation
- boto3을 사용해 s3에서 파일을 다운로드할 때 해당 에러가 발생한다
import boto3 FILE_PATH = 'image/atv_rider0001.jpg' BUCKET_NAME = 'flxr_yolo' KEY = 'image/atv_rider0001.jpg' client = boto3.client('s3') client.download_file(BUCKET_NAME, KEY, FILE_PATH)Solution
- Bucket의 이름이 잘못 표기되어 있었다..
- 정확한 이름 : flxr-yolo로 변경하니 제대로 작동한다.
- 다른 자료들을 찾아보니 대부분 경로 설정이나 오타와 같은 단순한 문제였다
Ref
1. //stackoverflow.com/questions/45109533/aws-cli-s3-copying-file-locally-using-the-terminal-fatal-error-an-error-occu
2. //stackoverflow.com/questions/46635895/aws-boto3-s3-python-an-error-occurred-404-when-calling-the-head object-operat?rq=1
'Computer Science > 에러 기록 정리' 카테고리의 다른 글
2020.06.28 |
2019.11.13 |
2019.11.13 |
2019.11.12 |
2019.11.12 |
2019.11.07 |
I'm trying to read s3 file from sagemaker notebook, but I got PermissionError: Forbidden and I don't know what I did wrong. I already set my s3 bucket as public.
---------------------------------------------------------------------------
ClientError Traceback (most recent call last)
~/anaconda3/envs/python3/lib/python3.6/site-packages/s3fs/core.py in info(self, path, version_id, refresh)
529 out = self._call_s3(self.s3.head_object, kwargs, Bucket=bucket,
--> 530 Key=key, **version_id_kw(version_id), **self.req_kw)
531 return {
~/anaconda3/envs/python3/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
199 **kwargs)
--> 200 return method(**additional_kwargs)
201
~/anaconda3/envs/python3/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358
~/anaconda3/envs/python3/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
675 error_class = self.exceptions.from_code(error_code)
--> 676 raise error_class(parsed_response, operation_name)
677 else:
ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden
During handling of the above exception, another exception occurred:
PermissionError Traceback (most recent call last)
<ipython-input-22-7b06c29b8c94> in <module>
----> 1 df = pd.read_csv(path)
~/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/io/parsers.py in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)
674 )
675
--> 676 return _read(filepath_or_buffer, kwds)
677
678 parser_f.__name__ = name
~/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds)
429 # See //github.com/python/mypy/issues/1297
430 fp_or_buf, _, compression, should_close = get_filepath_or_buffer(
--> 431 filepath_or_buffer, encoding, compression
432 )
433 kwds["compression"] = compression
~/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/io/common.py in get_filepath_or_buffer(filepath_or_buffer, encoding, compression, mode)
183
184 return s3.get_filepath_or_buffer(
--> 185 filepath_or_buffer, encoding=encoding, compression=compression, mode=mode
186 )
187
~/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/io/s3.py in get_filepath_or_buffer(filepath_or_buffer, encoding, compression, mode)
46 mode: Optional[str] = None,
47 ) -> Tuple[IO, Optional[str], Optional[str], bool]:
---> 48 file, _fs = get_file_and_filesystem(filepath_or_buffer, mode=mode)
49 return file, None, compression, True
~/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/io/s3.py in get_file_and_filesystem(filepath_or_buffer, mode)
27 fs = s3fs.S3FileSystem(anon=False)
28 try:
---> 29 file = fs.open(_strip_schema(filepath_or_buffer), mode)
30 except (FileNotFoundError, NoCredentialsError):
31 # boto3 has troubles when trying to access a public file
~/anaconda3/envs/python3/lib/python3.6/site-packages/fsspec/spec.py in open(self, path, mode, block_size, cache_options, **kwargs)
722 autocommit=ac,
723 cache_options=cache_options,
--> 724 **kwargs
725 )
726 if not ac:
~/anaconda3/envs/python3/lib/python3.6/site-packages/s3fs/core.py in _open(self, path, mode, block_size, acl, version_id, fill_cache, cache_type, autocommit, requester_pays, **kwargs)
376 version_id=version_id, fill_cache=fill_cache,
377 s3_additional_kwargs=kw, cache_type=cache_type,
--> 378 autocommit=autocommit, requester_pays=requester_pays)
379
380 def _lsdir(self, path, refresh=False, max_items=None):
~/anaconda3/envs/python3/lib/python3.6/site-packages/s3fs/core.py in __init__(self, s3, path, mode, block_size, acl, version_id, fill_cache, s3_additional_kwargs, autocommit, cache_type, requester_pays)
1095 self.req_kw = {'RequestPayer': 'requester'} if requester_pays else {}
1096 super().__init__(s3, path, mode, block_size, autocommit=autocommit,
-> 1097 cache_type=cache_type)
1098 self.s3 = self.fs # compatibility
1099 if self.writable():
~/anaconda3/envs/python3/lib/python3.6/site-packages/fsspec/spec.py in __init__(self, fs, path, mode, block_size, autocommit, cache_type, cache_options, **kwargs)
954 if mode == "rb":
955 if not hasattr(self, "details"):
--> 956 self.details = fs.info(path)
957 self.size = self.details["size"]
958 self.cache = caches[cache_type](
~/anaconda3/envs/python3/lib/python3.6/site-packages/s3fs/core.py in info(self, path, version_id, refresh)
546 return super(S3FileSystem, self).info(path)
547 else:
--> 548 raise ee
549 except ParamValidationError as e:
550 raise ValueError('Failed to head path %r: %s' % (path, e))
PermissionError: Forbidden