ai.awss3

Functions

Method	Parameters	Description
getFile(bucketName, key, region, awsAccessKey, AwsSecretKey)	bucketName: String key: String region: String awsAcceskey: String awsSecretKey: String	Retrieves a file from an AWS S3 bucket.
putFile(bucketName, key, region, awsAccessKey, AwsSecretKey)	bucketName: String key: String region: String awsAcceskey: String awsSecretKey: String	Puts a file into an AWS S3 bucket.
listKeys(bucketName, region, awsAccessKey, AwsSecretKey) Not recommended for buckets with over 5000 keys.	bucketName: String region: String awsAcceskey: String awsSecretKey: String	Lists the keys (files) in an AWS S3 bucket.
listObjects(request, region, awsAccessKey, AwsSecretKey)	Request object parameters See Amazon S3 Request Docs for details. bucketName: String (required). continuationToken: Indicates the list continues on this bucket with a token. The nextContinuationToken obtained from the previous response. delimiter: A character used to group keys. See Amazon S3 Listing Keys Hierarchically docs. encoding: The encoding method to use in the response. fetchOwner: Set to true to list owners with each key. If not specified the owner is null. maxKeys: The maximum number of keys in the response. prefix: Limits the response to keys that begin with the specified prefix. requestPlayer: String startAfter: The list begins after this key. Use any key in the bucket. Can't be set at the same time as the continuationToken. Response object parameters See Amazon S3 Response Docs for details. s3Objects: List of objects found. keyCount: Number of objects found. isTruncated: Indicates if more keys match the initial request. Can be specified in the request. AWS defaults to 1000. nextContinuationToken: Sent when isTruncated is true, indicating more keys in the bucket can be listed. The list can be continued with this nextContinuationToken. httpStatusCode: name: prefix: maxKeys: delimiter: commonPrefixes: encoding: continuationToken: startAfter:	Lists the objects in an AWS S3 bucket. Allows the use of parameters in a request to limit and filter the fetched keys.

Examples

// Manually invoke this method via 'Test connection'
function testConnection(context) {
    return true;
}
    
// Manually invoke this method via 'Run manually'
function exportData(context) {
    var bucketName = 'XXX';
    var key = 'XXX';
    var awsAccessKey = 'XXX';
    var awsSecretKey = 'XXX';
    var region = 'us-west-1';
    
    // Step 1: Get the data
    var reader = context.createTableReader();
        
    // Step 2: Process and export the data
    var result = "Column,Heading,Names\n";
    var row = null;
    while ((row = reader.readRow()) !== null) {
       // remove meta columns we don't need
       row.length = row.length - 2;
        
       result += row.join() + '\n';
   }
    
    ai.awss3.putFile(bucketName, key, result, region, awsAccessKey, awsSecretKey);
}

Example with listObjects

This example shows how to request objects and page through the results.

function processFiles(context) {
    // Retrieve access and secret keys from settings
    const awsAccessKey = context.getDataSource().getSetting('awsAccessKey').getValue();
    const awsSecretKey = context.getDataSource().getSetting('awsSecretKey').getValue();
    
    const region = 'us-west-1'    
    
    // Create the request for objects, at a minimum a bucket name is required
    const request = {
        BucketName: 'mybucket',
        Prefix: 'mydata/',            // Return keys from this directory
    };
    
    // In this example each of the key names will be pushed into this array
    const keyNames = [];
    
    let processing = true;
    while (processing)
    {
        var response = ai.awss3.listObjects(request, region, awsAccessKey, awsSecretKey);
        for (const entry of response.s3Objects)
        {
            // Here we have a single key that can be processed as desired
            keyNames.push(entry.key);
        }
        
        if (response.isTruncated)
        {
            // If the response did not include all remaining results, use the
            // nextContinuationToken from the response in the next request
            request.ContinuationToken = response.nextContinuationToken;
        }
        else
        {
            // If the response did include all remaining results, exit the loop
            ai.log.logInfo('Response indicated no further keys available');
            processing = false;            
        }
    }    
    // Log a comma separated list of keys    
    ai.log.logInfo(keyNames.join(', '));
}

Example parameter request indicating just BucketName, the only required value.

{
  BucketName: 'ag-ccds-test',
  Prefix: 'samplenumber/',
  StartAfter: 'samplenumber/001_samplefile',
  MaxKeys: 3
}

Example listObjects Response

This example listOjbects response lists 8 objects found.

{
  "isTruncated": true,
  "s3Objects": [
    {
      "eTag": "\"4c7507b8cf17a6650bcff6787ccd0915\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575352592693.txt",
      "lastModified": "2019-12-03T15:56:34+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    },
    {
      "eTag": "\"b2addf641c43f17df4492524c834984c\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575353843184.txt",
      "lastModified": "2019-12-03T16:17:24+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    },
    {
      "eTag": "\"b2addf641c43f17df4492524c834984c\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575353844190.txt",
      "lastModified": "2019-12-03T16:17:25+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    },
    {
      "eTag": "\"b2addf641c43f17df4492524c834984c\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575353845193.txt",
      "lastModified": "2019-12-03T16:17:26+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    },
    {
      "eTag": "\"b2addf641c43f17df4492524c834984c\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575353846195.txt",
      "lastModified": "2019-12-03T16:17:27+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    },
    {
      "eTag": "\"b2addf641c43f17df4492524c834984c\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575353847194.txt",
      "lastModified": "2019-12-03T16:17:28+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    },
    {
      "eTag": "\"b2addf641c43f17df4492524c834984c\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575353848223.txt",
      "lastModified": "2019-12-03T16:17:29+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    },
    {
      "eTag": "\"b2addf641c43f17df4492524c834984c\"",
      "bucketName": "ag-ccds-test",
      "key": "archiveListing/archiveList_1575353849192.txt",
      "lastModified": "2019-12-03T16:17:30+10:00",
      "owner": null,
      "size": 1323,
      "storageClass": {
        "value": "STANDARD"
      }
    }
  ],
  "name": "ag-ccds-test",
  "prefix": "archiveListing/",
  "maxKeys": 8,
  "commonPrefixes": [],
  "delimiter": null,
  "encoding": null,
  "keyCount": 8,
  "continuationToken": null,
  "nextContinuationToken": "1Vut/ORdOSPdn8ZQdd8Z5nhNvlEfPkw0hkekFIDTVzk8Hn8jlttn/yIjkl67yK3RAUkPIoUHwDuiybia7Rx7YjVaRefLocnF4",
  "startAfter": null,
  "responseMetadata": {
    "requestId": "5ACCAA141D2F7D2B",
    "metadata": {
      "x-amz-id-2": "zlslMcRM+qj6SCKELuE2crlEzhc0eEFkpo5OC3Pzn9tMvtOYnaEkLoRQNj1m1SThPzI1lQ233Zo="
    }
  },
  "contentLength": -1,
  "httpStatusCode": 200
}