Python：正在下载基于名称和日期的 s3 文件

Question

我正在尝试根据文件名的 ID 和日期从 s3 中提取文件：

命名约定：

命名规范如下：

** ID_NAME_DATE.csv : 文件名遵循同样的模式

示例：9919USEN_File_20180216.csv

示例：9919GBEN_File_20180211.csv

**

代码：

import boto3
import re

def downloadFiletest():

#connect to s3
client = boto3.resource(u's3', aws_access_key_id=u'KEY',
                       aws_secret_access_key=u'TOKEN')
                       
#used for downloading                      
s3 = boto3.client(u's3', aws_access_key_id=u'KEY',
                       aws_secret_access_key=u'TOKEN')
dateIdReg = '[0-9]{8}'
dateSuffix = re.compile(date)
print (u"= S3 Client Connected =")
# configure s3 bucket
bucket = client.Bucket(u'us-eu-Bucket')
b_folder = "/folder/example/"
c_folder = b_folder.lower() + '/'
files_not_found = True
for cList in bucket.objects.filter(Prefix=b_folder):
    cFiles= cList.key
    print ('file : ', cFiles)
    for fileId in cFiles.lower():
        files_not_found = False
        f = fileId.rstrip()
        print(f)
        fileidreg= '[0-9]{4}[a-zA-Z]{4}'
        FileID = re.compile(fileidreg)
        if FileID.match(f) and dateSuffix.match(f):
            print(u'cList.key.lower(): ', cList.key.lower())
            old_file = cList.key
            dot_index = old_file.find(u'.')
            print (u'old dot file name: ', dot_index)
            file_ext = old_file[dot_index:]
            cfile = fileId + '_file_' + dateSuffix + file_ext
            tmp_path = "/tmp/folder/" + cfile
            b_path = cVal + cfile
            print (u'b path : ', b_path)
            s3.download_file("us-eu-Bucket", b_path, tmp_path)
            print ("TEMP PATH: ", tmp_path)
            
    if files_not_found:
        print("ALERT", "No file in {0}/{1}".format(bucket, b_folder))

downloadFiletest()

错误：

它跳过 cFiles.lower() 中的 fileId: 并关闭脚本。

目标：

从 S3 中提取文件并将其下载到 tmp_path 以便根据需要使用。拉文件时，我希望脚本根据 ID 和日期选择文件。例如：

规则：伪:

如果S3有文件9919USEN_File_20180216.csv和9919USEN_File_20180217.csv则选择9919USEN_File_20180217.csv下载。另外 IF 991USEN_File_2018.csv in S3 然后不选择文件，因为它不匹配规则，fileidreg = '[0-9]{4}[a-zA-Z]{4}' 和 dateIdReg = '[ 0-9]{8}'.

规则：视觉：

9919USEN_File_20180217.csv > 9919USEN_File_20180216.csv [due to date] 9919USEN_File_20180217.csv > 991USEN_File_2018.csv [Due to Incorrect ID and Date]

Answer 1

解决方案

问题在于它的结构方式。我已经重新组织并把它放在一个 try 异常条件循环中。我还使用了 FileIDPrefix.search 而不是 FileIDPrefix.match 因为它只是专门查看索引并且不适合手头的问题。

最终解决方案。

import boto3
import re

#connect to s3
client = boto3.resource(u's3', aws_access_key_id=u'KEY',
                   aws_secret_access_key=u'TOKEN')

#used for downloading                      
s3 = boto3.client(u's3', aws_access_key_id=u'KEY',
                   aws_secret_access_key=u'TOKEN')

def downloadFiletest():
  date = '[0-9]{8}'  # fileDate regex
  dateSuffix = re.compile(dates)  # regex used to check the date of the file
  reg = '[0-9]{4}[a-zA-Z]{4}'  # filename regex
  fileIDPrefix = re.compile(reg)  # check fileID of the Filename.

  folder = u"/folder/example/"  # directory
  bucket = client.Bucket(bucketname)  # bucket

  try:
      for cuList in bucket.objects.filter(Prefix=folder):  # filter to the folder

          filenames= cList.key  # directory of the files that we would like to use
          print(cu)

          # specific locations of site fileID of the file and date of the file
          fileID = filenames[33:41]
          fileDate = filenames[51:59]

          # check the length of each values to be verified later.
          lenf = len(fileID)
          lenG = len(fileDate)
          old_file = cList.key
          dot_index = old_file.find(u'.')
          file_ext = old_file[dot_index:]

          # this check that the files in directory match our specified rules. if does it proceeds.
          if fileIDPrefix.search(cu) and fileDateSuffix.search(cu):
              filename = fileID + u'_file_' + fileDate + file_ext
              tmp_path = "/tmp/mpcmt/" + filename
              file_path = folder + filename
              s3.download_file(bucketname, file_path, tmp_path)


              return filename, tmp_path, fileID, fileDate

              # this check the number of values/char in a directory to see it matches up to what is expected.

          if dot_index > 59 or dot_index < 59:
                  print('File has wrong fileID or Wrong Date')
          if lenG > 8 or lenG < 8:
                  print('File has wrong fileDate Format')
          if lenf > 8 or lenf < 8:
                  print('File has wrong fileID')

  except Exception as e:  # this closes and displays an error if the file doesn't exist.
      print("ALERT", "No file in {0}/{1}".format(bucket, folder))
      # There was some issue / error / problem and that is why the program is exiting.
      print >> sys.stderr, "No file in {0}/{1}".format(bucket, folder)
      print >> sys.stderr, "Exception: %s" % str(e)
      sys.exit(1)


downloadFiletest()

Python：正在下载基于名称和日期的 s3 文件

Python: downloading s3 file based name and date

python

csv

amazon-s3

boto3

python-2.6