Inserting JSON into Amazon S3

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - INSERTING JSON INTO AMAZON S3

# Recipe URL - https://app.lingk.io/a/10932/tf/17873

# Description - # This recipe demonstrates output different JSON structures to Amazon S3.
                # To use this recipe, just click "Run"

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - S3Bucket
# Connectors - JSON, S3Bucket
# Data Flows - Single Direction
# Connection Type - JSON to S3 Bucket

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure S3Bucket credentials in your Environment before running this recipe
  # S3Bucket Setup - https://help.lingk.io/en/articles/298-amazon-s3-connector-setup-guide
  # JSON Setup - https://help.lingk.io/en/articles/74-json-connector-reference

###### Start: JSON Connectors #######

# Source data
-
    name: courseFile
    type: json
    properties:
      jsonObject: >
        [
          { "courseId": "phys15a", "creditHours": 5, "description": "Introduction to Mechanics", "accountId": "0014100000INNCQ" },
          { "courseId": "phys15b", "creditHours": 20, "description": "Introduction to E&M", "accountId": "0014100000INNCQ" },
          { "courseId": "phys15c", "creditHours": 10, "description": "Waves", "accountId": "0014100000INNCQ" },
          { "courseId": "blah001", "creditHours": 10, "blah": "Waves", "accountId": "0014100000INNCQ" }
        ]

###### End: JSON Connectors #######

###### Start: S3 Connectors #######

# Write the file out as JSON Lines
-
    name: courseS3WriterJsonl
    type: s3Writer
    format: json
    properties:
      s3ObjectKey: courses.jsonl

# Write the file out as a standard JSON Array
-
    name: courseS3WriterJsonArray
    type: s3Writer
    format: jsonArray
    properties:
      s3ObjectKey: courses.json

# Write the file out as nested JSON
-
    name: courseS3WriterNestedJson
    type: s3Writer
    format: json
    properties:
      s3ObjectKey: coursesNested.json

###### End: S3 Connectors #######

#   ______                         _       
#  |  ____|                       | |      
#  | |__ ___  _ __ _ __ ___   __ _| |_ ___ 
#  |  __/ _ \| '__| '_ ` _ \ / _` | __/ __|
#  | | | (_) | |  | | | | | | (_| | |_\__ \
#  |_|  \___/|_|  |_| |_| |_|\__,_|\__|___/

readFormats:
# write as JSON Lines
-
    name: json
    type: json

writeFormats:
# write as JSON Array
-
    name: jsonArray
    type: jsonArray

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

# this statement will write the file as a JSON Lines / NDJSON format
# one line per valid JSON object 
# this format is ideal for reading and writing large amounts of JSON
  - statement: INSERT courseFile INTO courseS3WriterJsonl

# this statement will write the file as a JSON Array
# one line per valid JSON object 
# this format is ideal for writing JSON in a standard Array that can be read by a browser 
  - statement: INSERT courseFile INTO courseS3WriterJsonArray

# To output all the records as a single array in a file, use the "collect_list" method to combine all records, and the struct method to make them JSON structures.
# This will create a single row and single column table. 
# The column name is "items"
# The value of items will be a JSON array [ {struct}, {struct},.. ]
  - statement: |
      (dataSetAsArray) => 
          SELECT collect_list(
                    struct(
                      courseId,
                      creditHours, 
                      description, 
                      accountId
                     )
                   ) AS items 
           FROM courseFile
  #- statement: PRINT dataSetAsArray

# The file will have a structure like
# { "items":[{"key":"value"},{"key":"value"}] }
# on a single line to be read by the browser easily
  - statement: INSERT dataSetAsArray INTO courseS3WriterCustomJson

# Add more statements to convert, join, aggregrate, transform, and integrate your data