こちらの記事を参考にdynamodbインポートを試みているのですがうまくいきません。
Amazon DynamoDB への CSV 一括取り込みの実装
https://aws.amazon.com/jp/blogs/news/implementing-bulk-csv-ingestion-to-amazon-dynamodb/
DynamoDB、Amazon S3、Lambda、および AWS CloudFormation にアクセスできる IAM ユーザー。
記事の内容はs3にcsvが入ったら、それをトリガーとしてcsvをdynamodbにインポートさせるlambdaを動かす
cloudfomationスタックを作るものと記憶しています。
dynamoDBの構成
テーブル名 test
プライマリパーティションキー tag (文字列)
プライマリソートキー keyword (文字列)
挿入csvの構成
hoge1,fuga1 hoge2,fuga2 hoge3,fuga3 hoge4,fuga4 hoge5,fuga5 hoge6,fuga6 hoge7,fuga7 hoge8,fuga8 hoge9,fuga9 hoge10,fuga10
手順としてfloudfomationに入り
1.スタックの作成をクリック
前提条件 - テンプレートの準備:
テンプレートの準備完了を選択
テンプレートの指定:
テンプレートファイルのアップロードを選択、
テンプレートには記事から入れるgithubにあるテンプレートを選択
2.スタックの詳細を指定
スタックの名前:csv-to-dynamodb
パラメータ
bucketname「dynamo-imp-0101」
dynamodbtablename[test]
↑サンプルに合うフォーマットで作成したテーブル
filename[test.csv]
↑サンプルファイルの上から2行のみ使用
3.スタックオプションの設定
アクセス許可:
以下のロールを使用
使用付与ロール:
s3fullaccess
DynamoDBfullaccess
Lambdafullaccess
CloudFormationfullaccess
この設定でスタックを作成しても、すぐにcloudfomationに
×マークがついてしまいます。
考えられるエラーでIAMの設定が適してない為、処理が継続しないことが
考えられます、ただし、十分にロールの権限を付与しているため
どのような権限を付与すればよいかわかりません。
「csvファイル名やテーブル名、バケット名が間違ってないか」
「csvとテーブルのフォーマット間違ってないか」
については調べてつぶしています。
イベントログはこちらになります。
2020-06-26 12:14:59 UTC+0900 testfile ROLLBACK_COMPLETE -
2020-06-26 12:14:58 UTC+0900 LambdaRole DELETE_COMPLETE -
2020-06-26 12:14:56 UTC+0900 DynamoDBTable DELETE_COMPLETE -
2020-06-26 12:14:56 UTC+0900 LambdaRole DELETE_IN_PROGRESS -
2020-06-26 12:14:49 UTC+0900 testfile ROLLBACK_IN_PROGRESS The following resource(s) failed to create: [DynamoDBTable, LambdaRole]. . Rollback requested by user.
2020-06-26 12:14:48 UTC+0900 LambdaRole CREATE_FAILED Resource creation cancelled
2020-06-26 12:14:47 UTC+0900 LambdaRole CREATE_IN_PROGRESS -
2020-06-26 12:14:47 UTC+0900 DynamoDBTable CREATE_FAILED testfile already exists in stack arn:aws:cloudformation:ap-northeast-1:671527041604:stack/xxx/b4e28760-b52a-11ea-9d7b-06c1e14cac70
{
"AWSTemplateFormatVersion": "2010-09-09",
"Metadata": {
},
"Parameters" : {
"BucketName": {
"Description": "Name of the S3 bucket you will deploy the CSV file to",
"Type": "String",
"ConstraintDescription": "must be a valid bucket name."
},
"FileName": {
"Description": "Name of the S3 file (including suffix)",
"Type": "String",
"ConstraintDescription": "Valid S3 file name."
},
"DynamoDBTableName": {
"Description": "Name of the dynamoDB table you will use",
"Type": "String",
"ConstraintDescription": "must be a valid dynamoDB name."
}
},
"Resources": {
"DynamoDBTable":{
"Type": "AWS::DynamoDB::Table",
"Properties":{
"TableName": {"Ref" : "test"},
"BillingMode": "PAY_PER_REQUEST",
"AttributeDefinitions":[
{
"AttributeName": "uuid",
"AttributeType": "S"
}
],
"KeySchema":[
{
"AttributeName": "uuid",
"KeyType": "HASH"
}
],
"Tags":[
{
"Key": "Name",
"Value": {"Ref" : "test"}
}
]
}
},
"LambdaRole" : {
"Type" : "AWS::IAM::Role",
"Properties" : {
"AssumeRolePolicyDocument": {
"Version" : "2012-10-17",
"Statement" : [
{
"Effect" : "Allow",
"Principal" : {
"Service" : ["lambda.amazonaws.com","s3.amazonaws.com"]
},
"Action" : [
"sts:AssumeRole"
]
}
]
},
"Path" : "/",
"ManagedPolicyArns":["arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole","arn:aws:iam::aws:policy/AWSLambdaInvocation-DynamoDB","arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"],
"Policies": [{
"PolicyName": "policyname",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"dynamodb:PutItem",
"dynamodb:BatchWriteItem"
]
}]
}
}]
}
},
"CsvToDDBLambdaFunction": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Handler": "index.lambda_handler",
"Role": {
"Fn::GetAtt": [
"LambdaRole",
"Arn"
]
},
"Code": {
"ZipFile": {
"Fn::Join": [
"\n",
[
"import json",
"import boto3",
"import os",
"import csv",
"import codecs",
"import sys",
"",
"s3 = boto3.resource('s3')",
"dynamodb = boto3.resource('dynamodb')",
"",
"bucket = os.environ['bucket']",
"key = os.environ['key']",
"tableName = os.environ['table']",
"",
"def lambda_handler(event, context):",
"",
"",
" #get() does not store in memory",
" try:",
" obj = s3.Object(bucket, key).get()['Body']",
" except:",
" print(\"S3 Object could not be opened. Check environment variable. \")",
" try:",
" table = dynamodb.Table(tableName)",
" except:",
" print(\"Error loading DynamoDB table. Check if table was created correctly and environment variable.\")",
"",
" batch_size = 100",
" batch = []",
"",
" #DictReader is a generator; not stored in memory",
" for row in csv.DictReader(codecs.getreader('utf-8')(obj)):",
" if len(batch) >= batch_size:",
" write_to_dynamo(batch)",
" batch.clear()",
"",
" batch.append(row)",
"",
" if batch:",
" write_to_dynamo(batch)",
"",
" return {",
" 'statusCode': 200,",
" 'body': json.dumps('Uploaded to DynamoDB Table')",
" }",
"",
"",
"def write_to_dynamo(rows):",
" try:",
" table = dynamodb.Table(tableName)",
" except:",
" print(\"Error loading DynamoDB table. Check if table was created correctly and environment variable.\")",
"",
" try:",
" with table.batch_writer() as batch:",
" for i in range(len(rows)):",
" batch.put_item(",
" Item=rows[i]",
" )",
" except:",
" print(\"Error executing batch_writer\")"
]
]
}
},
"Runtime": "python3.7",
"Timeout": 900,
"MemorySize": 3008,
"Environment" : {
"Variables" : {"bucket" : { "Ref" : "BucketName" }, "key" : { "Ref" : "FileName" },"table" : { "Ref" : "DynamoDBTableName" }}
}
}
},
"S3Bucket": {
"DependsOn" : ["CsvToDDBLambdaFunction","BucketPermission"],
"Type": "AWS::S3::Bucket",
"Properties": {
"BucketName": {"Ref" : "BucketName"},
"AccessControl": "BucketOwnerFullControl",
"NotificationConfiguration":{
"LambdaConfigurations":[
{
"Event":"s3:ObjectCreated:*",
"Function":{
"Fn::GetAtt": [
"CsvToDDBLambdaFunction",
"Arn"
]
}
}
]
}
}
},
"BucketPermission":{
"Type": "AWS::Lambda::Permission",
"Properties":{
"Action": "lambda:InvokeFunction",
"FunctionName":{"Ref" : "CsvToDDBLambdaFunction"},
"Principal": "s3.amazonaws.com",
"SourceAccount": {"Ref":"AWS::AccountId"}
}
}
},
"Outputs" : {
}
}