Go Back
Inserting JSON into Amazon S3
Publisher
: Lingk
Run In Lingk
Description
This recipe demonstrates output different JSON structures to Amazon S3. To use this recipe, just click "Run"
Browse the knowledge base
Twitter
E-Mail
# _____ _ _____ __ # | __ \ (_) |_ _| / _| # | |__) |___ ___ _ _ __ ___ | | _ __ | |_ ___ # | _ // _ \/ __| | '_ \ / _ \ | | | '_ \| _/ _ \ # | | \ \ __/ (__| | |_) | __/ _| |_| | | | || (_) | # |_| \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ # | | # |_| # Project Name - INSERTING JSON INTO AMAZON S3 # Recipe URL - https://app.lingk.io/a/10932/tf/17873 # Description - # This recipe demonstrates output different JSON structures to Amazon S3. # To use this recipe, just click "Run" # Industry - Higher Ed # Business Process - Graduate Reporting # Systems - S3Bucket # Connectors - JSON, S3Bucket # Data Flows - Single Direction # Connection Type - JSON to S3 Bucket # Add Recipe notes / Change log information here! # _____ _ # / ____| | | # | | ___ _ __ _ __ ___ ___| |_ ___ _ __ ___ # | | / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __| # | |___| (_) | | | | | | | __/ (__| || (_) | | \__ \ # \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_| |___/ # # CONNECTORS specify what data will be pulled into the in-memory database during processing connectors: # Configure S3Bucket credentials in your Environment before running this recipe # S3Bucket Setup - https://help.lingk.io/en/articles/298-amazon-s3-connector-setup-guide # JSON Setup - https://help.lingk.io/en/articles/74-json-connector-reference ###### Start: JSON Connectors ####### # Source data - name: courseFile type: json properties: jsonObject: > [ { "courseId": "phys15a", "creditHours": 5, "description": "Introduction to Mechanics", "accountId": "0014100000INNCQ" }, { "courseId": "phys15b", "creditHours": 20, "description": "Introduction to E&M", "accountId": "0014100000INNCQ" }, { "courseId": "phys15c", "creditHours": 10, "description": "Waves", "accountId": "0014100000INNCQ" }, { "courseId": "blah001", "creditHours": 10, "blah": "Waves", "accountId": "0014100000INNCQ" } ] ###### End: JSON Connectors ####### ###### Start: S3 Connectors ####### # Write the file out as JSON Lines - name: courseS3WriterJsonl type: s3Writer format: json properties: s3ObjectKey: courses.jsonl # Write the file out as a standard JSON Array - name: courseS3WriterJsonArray type: s3Writer format: jsonArray properties: s3ObjectKey: courses.json # Write the file out as nested JSON - name: courseS3WriterNestedJson type: s3Writer format: json properties: s3ObjectKey: coursesNested.json ###### End: S3 Connectors ####### # ______ _ # | ____| | | # | |__ ___ _ __ _ __ ___ __ _| |_ ___ # | __/ _ \| '__| '_ ` _ \ / _` | __/ __| # | | | (_) | | | | | | | | (_| | |_\__ \ # |_| \___/|_| |_| |_| |_|\__,_|\__|___/ readFormats: # write as JSON Lines - name: json type: json writeFormats: # write as JSON Array - name: jsonArray type: jsonArray # _____ _ _ _ # / ____| | | | | | # | (___ | |_ __ _| |_ ___ _ __ ___ ___ _ __ | |_ ___ # \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __| # ____) | || (_| | || __/ | | | | | __/ | | | |_\__ \ # |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/ # STATEMENTS specify how the data should be processed while in memory statements: #******************************************************************** D I S C L A I M E R *********************************************************************************************** # * # Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe. * # For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform * # * #******************************************************************** D I S C L A I M E R *********************************************************************************************** # this statement will write the file as a JSON Lines / NDJSON format # one line per valid JSON object # this format is ideal for reading and writing large amounts of JSON - statement: INSERT courseFile INTO courseS3WriterJsonl # this statement will write the file as a JSON Array # one line per valid JSON object # this format is ideal for writing JSON in a standard Array that can be read by a browser - statement: INSERT courseFile INTO courseS3WriterJsonArray # To output all the records as a single array in a file, use the "collect_list" method to combine all records, and the struct method to make them JSON structures. # This will create a single row and single column table. # The column name is "items" # The value of items will be a JSON array [ {struct}, {struct},.. ] - statement: | (dataSetAsArray) => SELECT collect_list( struct( courseId, creditHours, description, accountId ) ) AS items FROM courseFile #- statement: PRINT dataSetAsArray # The file will have a structure like # { "items":[{"key":"value"},{"key":"value"}] } # on a single line to be read by the browser easily - statement: INSERT dataSetAsArray INTO courseS3WriterCustomJson # Add more statements to convert, join, aggregrate, transform, and integrate your data
Create New Users in Canvas
Simple Lingk Event API Subscription Tester