Selecting Nested Data in Recipes

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - Selecting Nested Data in Recipes

# Recipe URL - https://app.lingk.io/a/10932/tf/18311

# Description -  This recipe shows a quick example of how to select nested data using dot notation.

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - 
# Connectors - JSON
# Data Flows - Single Direction
# Connection Type - Static Data

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure JSON connector in your Environment before running this recipe
  # JSON Setup - https://help.lingk.io/en/articles/74-json-connector-reference

# This JSON contains static values that is used as the source data to be processed
-
    name: students
    type: json
    properties:
      jsonObject: {"students":{"id":"1","names":[{"firstName":"John","lastName":"Thomas","fullname":"John Thomas"}],"email":"jthomas@gmail.com","credentials":[{"password":"AEIOU8901"},{"password":"pass123!"},{"password":"ABC123"}]}}

-
    name: programs
    type: json
    properties:
      jsonObject: {"programs":{"id":"3456","code":"11234SAFFSA","title":"engineering"}}

-
    name: studentProgram
    type: json
    properties:
      jsonObject: {"studentProgram":{"programId":"3456","student":"1","enrollmentStatus":[{"id":"2345","status":"deactivated"}]}}

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

- statement: (getStudent) => select * from students
  - statement: print getStudent
  - statement: (getStudentCount) => select count(*) from students
  - statement: print getStudentCount

- statement: (getProgram) => select * from programs
  - statement: print getProgram
  - statement: (getProgramCount) => select count(*) from programs
  - statement: print getProgramCount

- statement: (getStudentProgram) => select * from studentProgram
  - statement: print getStudentProgram
  - statement: (getStudentProgramCount) => select count(*) from studentProgram
  - statement: print getStudentProgramCount

# explode function in the query below splits the students array in different rows and creates a lateral view
  - statement: |
      (bannerCredentials) => select 
                            students.id student_id,
                            a.credentials.*
                            from students                       
                            lateral view explode(students.credentials) a as credentials 
  - statement: print bannerCredentials

# Note that the dot notation and '[0]' enables you to choose a value from an array. 
  # Data in studentReader contains different fields retrieved as an arrays, that's the reason on why some of the items selected in the query below specifies the element [0]
  # The 'INNER JOIN' selects all rows from both relations where there is match (getStudent table and the following ones) 
  - statement: |
      (joinedDataset) => 
        select 
               students.id                               student_id,
               students.names[0].fullname                full_name,
               students.names[0].firstName               first_name,
               students.names[0].lastName                last_name,
               programs.id                               program_id,
               programs.code                             program_code,
               programs.title                            program_title,
               studentProgram.enrollmentStatus[0].status   program_status
        from getStudent students
          INNER JOIN getStudentProgram studentProgram on studentProgram.student = students.id
          INNER JOIN getProgram programs on programs.id = studentProgram.programId          
          INNER JOIN bannerCredentials on bannerCredentials.student_id = studentProgram.student

- statement: print joinedDataset
  - statement: (joinedDatasetCount) => select count(*) from joinedDataset
  - statement: print joinedDatasetCount

# Add more statements to convert, join, aggregrate, transform, and integrate your data