Created
September 28, 2022 18:38
-
-
Save rafael-gumiero/5b1e729e66f5c4a868153a953aa467d7 to your computer and use it in GitHub Desktop.
es-snapshot-stack.yml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| AWSTemplateFormatVersion: "2010-09-09" | |
| Parameters: | |
| ESBackupBucket: | |
| Type: String | |
| ESDomainArn: | |
| Type: String | |
| ESDomainEndpoint: | |
| Type: String | |
| myVPC: | |
| Type: AWS::EC2::VPC::Id | |
| mySubnets: | |
| Type: List<AWS::EC2::Subnet::Id> | |
| Resources: | |
| S3Bucket: | |
| Type: AWS::S3::Bucket | |
| Properties: | |
| BucketName: !Ref ESBackupBucket | |
| ESExecuteBackupRole: | |
| Type: "AWS::IAM::Role" | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: "2012-10-17" | |
| Statement: | |
| - | |
| Effect: "Allow" | |
| Principal: | |
| Service: | |
| - "es.amazonaws.com" | |
| Action: | |
| - "sts:AssumeRole" | |
| Path: "/" | |
| ESExecuteBackupPolicy: | |
| Type: "AWS::IAM::Policy" | |
| Properties: | |
| PolicyName: !Sub '${AWS::StackName}-es-backup-Policy' | |
| PolicyDocument: | |
| Version: "2012-10-17" | |
| Statement: | |
| - Effect: "Allow" | |
| Action: | |
| - "s3:GetObject" | |
| - "s3:PutObject" | |
| - "s3:DeleteObject" | |
| Resource: | |
| - !Sub 'arn:aws:s3:::${ESBackupBucket}/*' | |
| - Effect: "Allow" | |
| Action: | |
| - "s3:ListBucket" | |
| Resource: | |
| - !Sub 'arn:aws:s3:::${ESBackupBucket}' | |
| Roles: | |
| - | |
| Ref: "ESExecuteBackupRole" | |
| ESBackupInstanceProfile: | |
| Type: "AWS::IAM::InstanceProfile" | |
| Properties: | |
| Path: "/" | |
| Roles: | |
| - | |
| Ref: "ESExecuteBackupRole" | |
| LambdaExecuteBackRole: | |
| Type: "AWS::IAM::Role" | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: "2012-10-17" | |
| Statement: | |
| - | |
| Effect: "Allow" | |
| Principal: | |
| Service: | |
| - "lambda.amazonaws.com" | |
| Action: | |
| - "sts:AssumeRole" | |
| Path: "/" | |
| LambdaRolePolicies: | |
| Type: "AWS::IAM::Policy" | |
| Properties: | |
| PolicyName: !Sub '${AWS::StackName}-lambda-bkp-policy' | |
| PolicyDocument: | |
| Version: "2012-10-17" | |
| Statement: | |
| - Effect: "Allow" | |
| Action: | |
| - "iam:PassRole" | |
| Resource: !GetAtt ESExecuteBackupRole.Arn | |
| - Effect: "Allow" | |
| Action: | |
| - "es:ESHttpPut" | |
| - "es:ESHttpGet" | |
| - "es:ESHttpPost" | |
| Resource: | |
| - !Sub '${ESDomainArn}/*' | |
| - !Sub '${ESDomainArn}' | |
| - Effect: "Allow" | |
| Action: | |
| - "logs:CreateLogGroup" | |
| - "logs:CreateLogStream" | |
| - "logs:PutLogEvents" | |
| Resource: '*' | |
| - Effect: "Allow" | |
| Action: | |
| - "ec2:DescribeNetworkInterfaces" | |
| - "ec2:CreateNetworkInterface" | |
| - "ec2:DeleteNetworkInterface" | |
| - "ec2:DescribeInstances" | |
| - "ec2:AttachNetworkInterface" | |
| Resource: '*' | |
| Roles: | |
| - | |
| Ref: "LambdaExecuteBackRole" | |
| LambdaInstanceProfile: | |
| Type: "AWS::IAM::InstanceProfile" | |
| Properties: | |
| Path: "/" | |
| Roles: | |
| - | |
| Ref: "LambdaExecuteBackRole" | |
| LambdaSecurityGroup: | |
| Type: AWS::EC2::SecurityGroup | |
| Properties: | |
| GroupDescription: Allow http to client host | |
| VpcId: | |
| Ref: myVPC | |
| SecurityGroupEgress: | |
| - IpProtocol: tcp | |
| FromPort: 0 | |
| ToPort: 65535 | |
| CidrIp: 0.0.0.0/0 | |
| LambdaRestoreSnapshot: | |
| Type: AWS::Lambda::Function | |
| DependsOn: LambdaRolePolicies | |
| Properties: | |
| Runtime: nodejs12.x | |
| Timeout: 600 | |
| Role: !GetAtt LambdaExecuteBackRole.Arn | |
| Handler: index.handler | |
| Environment: | |
| Variables: | |
| REGION: !Ref AWS::Region | |
| DOMAIN_ENDPOINT: !Ref ESDomainEndpoint | |
| BUCKET_NAME: !Ref ESBackupBucket | |
| ROLE_ARN: !GetAtt ESExecuteBackupRole.Arn | |
| VpcConfig: | |
| SecurityGroupIds: | |
| - !Ref LambdaSecurityGroup | |
| SubnetIds: !Ref mySubnets | |
| Code: | |
| ZipFile: | | |
| var AWS = require('aws-sdk'); | |
| exports.handler = function(event, context, callback) { | |
| const domainEndpoint = process.env.DOMAIN_ENDPOINT; | |
| const region = process.env.REGION; | |
| var creds = new AWS.EnvironmentCredentials('AWS'); | |
| var endpoint = new AWS.Endpoint(domainEndpoint); | |
| // more options on restore | |
| //https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshots-restore-snapshot.html | |
| // let indices = ""; //if you want to restore all indices, uncomment this | |
| let indices = { | |
| "indices": "index1,index2", | |
| "ignore_unavailable": false, | |
| "include_global_state": false | |
| } | |
| let repoName = "es-snapshot-repo-2021-3"; //replace here with your repoName | |
| let snapshotName = "backup-2021-3-31-21-50-4"; //replace here with your snapshot | |
| var req = new AWS.HttpRequest(endpoint); | |
| req.method = 'POST'; | |
| req.path = `/_snapshot/${repoName}/${snapshotName}/_restore`; | |
| if(indices){ | |
| req.body = JSON.stringify(indices); | |
| } | |
| req.region = region; | |
| req.headers['presigned-expires'] = false; | |
| req.headers['Host'] = endpoint.host; | |
| req.headers['Content-Type'] = "application/json;charset=UTF-8"; | |
| var signer = new AWS.Signers.V4(req, 'es'); | |
| signer.addAuthorization(creds, new Date()); | |
| var send = new AWS.HttpClient(); | |
| send.handleRequest(req, null, function(httpResp) { | |
| var respBody = ''; | |
| httpResp.on('data', function (chunk) { | |
| respBody += chunk; | |
| }); | |
| httpResp.on('end', async function (chunk) { | |
| console.log('Response: ' + respBody); | |
| var resp = JSON.parse(respBody); | |
| }); | |
| }, function(err) { | |
| console.log('Error: ' + err); | |
| context.fail('Failed to query Elastic Search ' + err); | |
| }); | |
| }; | |
| LambdaTakeSnapshot: | |
| Type: AWS::Lambda::Function | |
| DependsOn: LambdaRolePolicies | |
| Properties: | |
| Runtime: nodejs12.x | |
| Timeout: 600 | |
| Role: !GetAtt LambdaExecuteBackRole.Arn | |
| Handler: index.handler | |
| Environment: | |
| Variables: | |
| REGION: !Ref AWS::Region | |
| DOMAIN_ENDPOINT: !Ref ESDomainEndpoint | |
| BUCKET_NAME: !Ref ESBackupBucket | |
| ROLE_ARN: !GetAtt ESExecuteBackupRole.Arn | |
| VpcConfig: | |
| SecurityGroupIds: | |
| - !Ref LambdaSecurityGroup | |
| SubnetIds: !Ref mySubnets | |
| Code: | |
| ZipFile: | | |
| var AWS = require('aws-sdk'); | |
| const domainEndpoint = process.env.DOMAIN_ENDPOINT; | |
| const bucketName = process.env.BUCKET_NAME; | |
| const region = process.env.REGION; | |
| const roleArn = process.env.ROLE_ARN; | |
| exports.handler = async function(event, context, callback) { | |
| try { | |
| const clusterStatus = await getClusterStatus(); | |
| if(clusterStatus == "red"){ | |
| console.log(`Cluster not health (status ${clusterStatus})- skpping snapshot`); | |
| return "cluster unhealthy - snapshot not taken"; | |
| } | |
| var d = new Date(); | |
| const repoName = `my-es-snapshot-repo-${d.getFullYear()}-${d.getMonth()+1}` | |
| console.log(`Repository name: ${repoName}`) | |
| await checkAndCreateMonthlyRepo(repoName); | |
| const backupName = `${repoName}/backup-${d.getFullYear()}-${d.getMonth()+1}-${d.getDate()}-${d.getHours()}-${d.getMinutes()}-${d.getSeconds()}` | |
| console.log(`Taking backup with name: ${backupName}`) | |
| const result = await takeSnapshot(backupName); | |
| console.log(`backup successfully taken - ${backupName}`) | |
| console.log(result); | |
| return result; | |
| } catch (e) { | |
| console.log(e.err) | |
| console.log(e) | |
| return e; | |
| } | |
| }; | |
| async function getClusterStatus(){ | |
| const clusterHealth = await myHttpRequest("",domainEndpoint,'GET', "/_cluster/health"); | |
| return clusterHealth.status; | |
| } | |
| async function takeSnapshot(backupName){ | |
| return await myHttpRequest("",domainEndpoint,'PUT', "/_snapshot/"+backupName); | |
| } | |
| async function checkAndCreateMonthlyRepo(repoName){ | |
| let snapshots= await myHttpRequest("",domainEndpoint,'GET', '/_snapshot'); | |
| console.log(`Checking if repository ${repoName} has been registered before`) | |
| if(!snapshots[repoName]){ | |
| console.log("repo doesn't exist - creating repo "+repoName); | |
| let body = { | |
| "type": "s3", | |
| "settings": { | |
| "bucket": bucketName, | |
| "region": region, | |
| "role_arn": roleArn, | |
| "base_path": repoName | |
| } | |
| } | |
| const result = await myHttpRequest(body,domainEndpoint,'PUT', "/_snapshot/"+repoName); | |
| console.log(`repository created successfully - ${repoName}`) | |
| console.log(result); | |
| } else { | |
| console.log(`${repoName} already exists - skipping`) | |
| } | |
| } | |
| const myHttpRequest = async function (body, domain, method, path) { | |
| return new Promise((resolve, reject) => { | |
| var endpoint = new AWS.Endpoint(domain); | |
| var request = new AWS.HttpRequest(endpoint, process.env.REGION); | |
| request.method = method; | |
| request.path = path; | |
| request.headers['host'] = domain; | |
| request.headers['Content-Type'] = 'application/json'; | |
| if (body){ | |
| request.body = JSON.stringify(body); | |
| } | |
| var credentials = new AWS.EnvironmentCredentials('AWS'); | |
| var signer = new AWS.Signers.V4(request, 'es'); | |
| signer.addAuthorization(credentials, new Date()); | |
| var client = new AWS.HttpClient(); | |
| const req = client.handleRequest(request, null, res => { | |
| let buffer = ""; | |
| res.on('data', chunk => buffer += chunk) | |
| res.on('end', () => { | |
| if(res.statusCode == "200" || res.statusCode == "201") { | |
| resolve(JSON.parse(buffer)); | |
| } else { | |
| reject(JSON.parse(buffer)); | |
| } | |
| }); | |
| }); | |
| req.on('error', e => reject(e.message)); | |
| }); | |
| } | |
| ScheduledRule: | |
| Type: AWS::Events::Rule | |
| Properties: | |
| Description: "ScheduledRule" | |
| ScheduleExpression: "rate(60 minutes)" | |
| State: "ENABLED" | |
| Targets: | |
| - | |
| Arn: | |
| Fn::GetAtt: | |
| - "LambdaTakeSnapshot" | |
| - "Arn" | |
| Id: "TargetFunctionV1" | |
| PermissionForEventsToInvokeLambda: | |
| Type: AWS::Lambda::Permission | |
| Properties: | |
| FunctionName: !Ref "LambdaTakeSnapshot" | |
| Action: "lambda:InvokeFunction" | |
| Principal: "events.amazonaws.com" | |
| SourceArn: | |
| Fn::GetAtt: | |
| - "ScheduledRule" | |
| - "Arn" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment