Ellucian Ethos to Salesforce EDA ( Terms )

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name -   ELLUCIAN ETHOS TO SALESFORCE EDA ( TERMS )

# Recipe URL - https://app.lingk.io/a/10932/tf/17457

# Description - This recipe pulls academic period, Term data, from Ellucian Ethos to load into Salesforce EDA Term Object.

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - Ethos, Salesforce
# Connectors - Ethos, Salesforce
# Data Flows - Single Direction
# Connection Type - API to API

# Data Models
## Ethos Models Used:
#1. academic-periods

## Salesforce Objects Used:
#1. Account
#2. hed__Term__c

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure Ethos & Salesforce credentials in your Environment before running this recipe
  # Ethos Setup - https://help.lingk.io/en/articles/292-ellucian-ethos-connector-setup-guide
  # Salesforce Setup - https://help.lingk.io/en/articles/267-salesforce-connector-setup-guide

###### Start - Ethos Reader connectors ######

-
    name: ethosAcademicPeriodsReader
    type: ethosReader
    parameterizedBy: FilterdTermStartDateInfo
    delayedRead: true
    properties:
      path: /api/academic-periods?criteria=%7B%22startOn%22%3A%7B%22%24gte%22%3A%22{{var.date}}%22%7D%7D
      dataModelVersion: 16.1.0 
    schema:
      fields:
        - name: 'id'
          type: 'string' 
        - name: 'title'
          type: 'string' 
        - name: 'code'
          type: 'string' 
        - name: 'startOn'
          type: 'string'
        - name: 'endOn'
          type: 'string'  
        - name: 'registration'
          type: 'string'  
        - name: 'category'
          type: 'object'
          objectSchema:
            name: objectSchemaSchema
            fields:
              - name: 'type'
                type: 'string'
              - name: 'parent'
                type: 'object'
                objectSchema:
                  name: objectSchemaSchema
                  fields:
                    - name: 'academicYear'
                      type: 'string'  
              - name: 'preceding'
                type: 'object'
                objectSchema:
                  name: objectSchemaSchema
                  fields:
                    - name: 'id'
                      type: 'string'

###### Start - Ethos Reader connectors ######

###### Start - Salesforce Reader connectors ######
-
    name: sfUniversityInfo
    type: salesforceReader
    delayedRead: true
    properties:
      useBearerToken: true
      query: 
        SELECT 
          Id,
          Name
        FROM Account
        WHERE Name = '{{env.vars.universityname}}'
        LIMIT 1
    schema:
      fields: 
        - name: 'Id'
          type: 'string'
        - name: 'Name'
          type: 'string'

- 
    name: sfhedTermReader
    type: salesforceReader
    delayedRead: true
    properties: 
      useBearerToken: true
      query: SELECT Id, External_GUID__c FROM hed__Term__c WHERE External_GUID__c != NULL 
    schema:
      fields:
        - name: 'Id'
          type: 'String' 
        - name: 'External_GUID__c'
          type: 'String'

###### End - Salesforce Reader connectors ######

###### Start - Salesforce Writer connectors ######

-
    name: sfhedTermWriter
    type: salesforceBulkWriter
    properties:
      onError: continue
      sfObject: hed__Term__c   
      batchSize: 200
      useBearerToken: true

###### End - Salesforce Writer connectors ######

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

## Get date before 3 months from today's date
  - statement: (FilterdTermStartDateInfo) => SELECT date_format(add_months(current_date(), -3), "yyyy-MM-dd'T'HH:mm:ss'Z'") AS date
  #- statement: print FilterdTermStartDateInfo

- statement: REFRESH ethosAcademicPeriodsReader
  - statement: (ethosAcademicPeriodsReaderCount) => SELECT COUNT(*) FROM ethosAcademicPeriodsReader
  #- statement: print ethosAcademicPeriodsReaderCount

# CASE function works as a dictionary and retrieve a specific value depending to the type field value pulled from the table as long as it is not null.
  # TRIM - Removes the leading and trailing space characters from str.
  # The 'ifnull' function retrieves the value of the second parameter in case the value of the first one is NULL, otherwise it will retrieve the current value of it
  - statement: |
        (ethosAcademicTermInfo) => SELECT
                                    id                                          `External_GUID__c`,
                                    title                                       `Name`,
                                    code                                        `code`,
                                    date_format(to_date(startOn), 'yyyy-MM-dd') `hed__Start_Date__c`,
                                    date_format(to_date(endOn), 'yyyy-MM-dd')   `hed__End_Date__c`,

CASE WHEN IFNULL(TRIM(upper(category.type)), '') = "YEAR" THEN "School Year"
                                    WHEN IFNULL(TRIM(upper(category.type)), '') = "TERM" THEN "Semester"
                                    WHEN IFNULL(TRIM(upper(category.type)), '') = "SUBTERM" THEN "Part of Term"
                                    ELSE '' END                                 `hed__Type__c`,

category.parent.academicYear                `category_academicYear`,
                                    category.preceding.id                       `category_preceding_id`

FROM ethosAcademicPeriodsReader

#- statement: print ethosAcademicTermInfo
  - statement: (ethosAcademicTermInfoCount) => SELECT COUNT(*) FROM ethosAcademicTermInfo
  #- statement: print ethosAcademicTermInfoCount

- statement: REFRESH sfhedTermReader
  - statement: (sfhedTermReaderCount) => SELECT COUNT(*) FROM sfhedTermReader
  #- statement: print sfhedTermReaderCount

#### Start - Update hed__Term__c ####

- statement: |
        (hedTermToUpdate) => 
              SELECT 
                  sf.Id,  
                  term.Name,
                  term.hed__Start_Date__c,
                  term.hed__End_Date__c,
                  term.hed__Type__c

FROM ethosAcademicTermInfo term
                INNER JOIN sfhedTermReader sf ON term.External_GUID__c = sf.`External_GUID__c`

- statement: (hedTermToUpdateCount) => SELECT COUNT(*) FROM hedTermToUpdate
  #- statement: print hedTermToUpdateCount
  - statement: UPDATE hedTermToUpdate INTO sfhedTermWriter

#### End - Update hed__Term__c ####

- statement: REFRESH sfUniversityInfo
  #- statement: print sfUniversityInfo

#### Start - Insert hed__Term__c ####

- statement: |
        (hedTermToInsert) => 
              SELECT    
                  term.External_GUID__c,
                  term.Name,
                  term.hed__Start_Date__c,
                  term.hed__End_Date__c,
                  term.hed__Type__c,

(SELECT Id FROM sfUniversityInfo)      `hed__Account__c`

FROM ethosAcademicTermInfo term
                LEFT ANTI JOIN sfhedTermReader sf ON term.External_GUID__c = sf.`External_GUID__c`

- statement: (hedTermToInsertCount) => SELECT COUNT(*) FROM hedTermToInsert
  #- statement: print hedTermToInsertCount

- statement: INSERT hedTermToInsert INTO sfhedTermWriter

#### End - Insert hed__Term__c ####

# Add more statements to convert, join, aggregrate, transform, and integrate your data