Ellucian Ethos to Salesforce EDA ( Registration )

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name -  ELLUCIAN ETHOS TO SALESFORCE EDA ( REGISTRATION )

# Recipe URL - https://app.lingk.io/a/10932/tf/17451

# Description - This recipe retrieves Ellucian Ethos Registration and writes back to Salesforce custom object (hed__Course_Enrollment__c)

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - Ethos, Salesforce
# Connectors - Ethos, Salesforce
# Data Flows - Single Direction
# Connection Type - API to API

# Data Models
## Ethos Models Used:
#1. section-registrations
#2. student-grade-point-averages

## Salesforce Objects Used:
#1. hed__Term__c
#2. Contact
#3. hed__Course_Offering__c
#4. hed__Program_Enrollment__c
#5. RecordType
#6. hed__Course_Enrollment__c

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure Ethos & Salesforce credentials in your Environment before running this recipe
  # Ethos Setup - https://help.lingk.io/en/articles/292-ellucian-ethos-connector-setup-guide
  # Salesforce Setup - https://help.lingk.io/en/articles/267-salesforce-connector-setup-guide

###### Start - Ethos Reader connectors ######

-
   name: ethosSectionRegistrationsReader
   type: ethosReader
   parameterizedBy: sfhedTermReader
   delayedRead: true
   properties:
     path: '/api/section-registrations?academicPeriod=%7B%22academicPeriod%22%3A%7B%22id%22%3A%22{{var.External_GUID__c}}%22%7D%7D'
     dataModelVersion: 16
   schema:
     fields:
      - name: 'id'
        type: 'string'
      - name: 'involvement'
        type: 'object'
        objectSchema:
          name: objectSchemaSchema
          fields:
            - name: 'endOn'
              type: 'string'
            - name: 'startOn'
              type: 'string'
      - name: 'registrant'
        type: 'object'
        objectSchema:
          name: objectSchemaSchema
          fields:
            - name: 'id'
              type: 'string'
      - name: 'section'
        type: 'object'
        objectSchema:
          name: objectSchemaSchema
          fields:
            - name: 'id'
              type: 'string'

-
    name: ethosStudentGPAReader
    type: ethosReader
    parameterizedBy: sectionInformation
    delayedRead: true
    properties:
      path: '/api/student-grade-point-averages?criteria=%7B%22student%22%3A%20%7B%22id%22%3A%20%22{{var.person_id}}%22%7D%7D'
      dataModelVersion: 1
    schema:
      fields:
        - name: 'student'
          type: 'object'
          objectSchema:
            name: objectSchemaSchema
            fields:
              - name: 'id'
                type: 'string'
        - name: 'periodBased'
          type: 'array'
          items:
            name: 'arrayElement'
            type: 'object'
            objectSchema:
              name: objectSchemaSchema
              fields:
                - name: 'academicPeriod'
                  type: 'object'
                  objectSchema:
                    name: objectSchemaSchema
                    fields:
                      - name: 'id'
                        type: 'string'
                - name: 'value'
                  type: 'string'
                - name: 'attemptedCredits'
                  type: 'string'
                - name: 'earnedCredits'
                  type: 'string'  
                - name: 'qualityPoints'
                  type: 'string'

###### End - Ethos Reader connectors ######

###### Start - Salesforce Reader connectors ######

- 
    name: sfhedTermReader
    type: salesforceReader
    parameterizedBy: FilterdTermStartDateInfo
    delayedRead: true
    properties: 
      useBearerToken: true
      query: SELECT Id, External_GUID__c, hed__Start_Date__c FROM hed__Term__c WHERE hed__Start_Date__c >= {{var.date}} LIMIT 5
    schema:
      fields:
        - name: 'Id'
          type: 'String' 
        - name: 'External_GUID__c'
          type: 'String' 
        - name: 'hed__Start_Date__c'
          type: 'String'

- 
    name: sfContactReader
    type: salesforceReader
    parameterizedBy: ethosSectionRegistrationInfo
    delayedRead: true
    properties: 
      useBearerToken: true
      query: SELECT Id, External_GUID__c FROM Contact WHERE External_GUID__c = '{{var.person_id}}' 
    schema:
      fields:
        - name: 'Id'
          type: 'string' 
        - name: 'External_GUID__c'
          type: 'string'

- 
    name: sfhedCourseOfferingReader
    type: salesforceReader
    delayedRead: true
    properties: 
      useBearerToken: true
      query: SELECT Id, External_GUID__c, hed__Term__r.External_GUID__c FROM hed__Course_Offering__c WHERE External_GUID__c != NULL
    schema:
      fields:
        - name: 'Id'
          type: 'String' 
        - name: 'External_GUID__c'
          type: 'String'          
        - name: 'hed__Term__r.External_GUID__c'
          type: 'String'

- 
    name: sfhedProgramEnrollmentReader
    type: salesforceReader
    delayedRead: true
    properties: 
      useBearerToken: true
      query: SELECT Id, hed__Contact__c, hed__Account__c FROM hed__Program_Enrollment__c
    schema:
      fields:
        - name: 'Id'
          type: 'string'
        - name: 'hed__Contact__c'
          type: 'string'  
        - name: 'hed__Account__c'
          type: 'string'

- 
    name: sfRecordTypeReader
    type: salesforceReader
    delayedRead: true
    properties: 
      useBearerToken: true
      query: SELECT Id, Name FROM RecordType WHERE sObjectType = 'hed__Course_Enrollment__c' AND Name = 'Student' LIMIT 1
    schema:
      fields:
        - name: 'Id'
          type: 'string'
        - name: 'Name'
          type: 'string'

- 
    name: sfhedCourseEnrollmentReader
    type: salesforceReader
    delayedRead: true
    properties: 
      useBearerToken: true
      query: SELECT Id, hed__Contact__c, hed__Course_Offering__c FROM hed__Course_Enrollment__c
    schema:
      fields:
        - name: 'Id'
          type: 'string'
        - name: 'hed__Contact__c'
          type: 'string'  
        - name: 'hed__Course_Offering__c'
          type: 'string'

###### End - Salesforce Reader connectors ######

###### Start - Salesforce Writer connectors ######          
-
    name: sfCourseEnrollmentWriter
    type: salesforceBulkWriter
    properties:
      onError: continue
      sfObject: hed__Course_Enrollment__c
      batchSize: 200
      useBearerToken: true

###### End - Salesforce Writer connectors ######

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

## Get date before 3 months from today's date
  - statement: (FilterdTermStartDateInfo) => SELECT date_format(add_months(current_date(), -3), "yyyy-MM-dd") AS date
  #- statement: print FilterdTermStartDateInfo

- statement: REFRESH sfhedTermReader
  - statement: (sfhedTermReaderCount) => SELECT COUNT(*) FROM sfhedTermReader
  #- statement: print sfhedTermReaderCount

- statement: REFRESH ethosSectionRegistrationsReader
  - statement: (ethosSectionRegistrationsReaderCount) => SELECT COUNT(*) FROM ethosSectionRegistrationsReader
  #- statement: print ethosSectionRegistrationsReaderCount

# date functions:
   # First - converts the person dates (startOn and endOn) from String into a Date formate using the "to_date" function
   # Then - through "date_format" function it provides the format needed to the date retrieved
  - statement: |
      (ethosSectionRegistrationInfo) =>
          SELECT
            id                                              `ethos_id`,
            registrant.id                                   `person_id`,
            section.id                                      `section_id`,
            date_format(to_date(involvement.startOn), 'yyyy-MM-dd') 
                                                            `involvement_start_date`,
            date_format(to_date(involvement.endOn), 'yyyy-MM-dd')
                                                            `involvement_end_date`,
            date_format(current_date(), 'yyyy-MM-dd')       `today_date`

FROM ethosSectionRegistrationsReader

#- statement: print ethosSectionRegistrationInfo

- statement: REFRESH sfContactReader
  - statement: (sfContactReaderCount) => SELECT COUNT(*) FROM sfContactReader
  #- statement: print sfContactReaderCount

- statement: REFRESH sfhedCourseOfferingReader
  - statement: (sfhedCourseOfferingReaderCount) => SELECT COUNT(*) FROM sfhedCourseOfferingReader
  #- statement: print sfhedCourseOfferingReaderCount

# CASE function is used as a dictionary and depending on the value received as a STRING, is the new status value that will be retrieved now as a BOOLEAN
  - statement: |
      (sectionInformation) =>
          SELECT
            s.ethos_id                                        `ethos_id`,
            s.person_id                                       `person_id`,

CASE WHEN s.today_date <= s.involvement_end_date THEN "Current"
            ELSE "Former" END                                 `hed__Status__c`,

c.Id                                              `hed__Contact__c`,
            off.Id                                            `hed__Course_Offering__c`

FROM ethosSectionRegistrationInfo s
          INNER JOIN sfContactReader c ON s.person_id = c.External_GUID__c
          INNER JOIN sfhedCourseOfferingReader off ON s.section_id = off.External_GUID__c

#- statement: print sectionInformation
  - statement: (sectionInformationCount) => SELECT COUNT(*) FROM sectionInformation
  #- statement: print sectionInformationCount

- statement: REFRESH ethosStudentGPAReader
  - statement: (ethosStudentGPAReaderCount) => SELECT COUNT(*) FROM ethosStudentGPAReader
  #- statement: print ethosStudentGPAReaderCount

# explode function in the query below splits the phones array in different rows and creates a lateral view
  - statement: |
      (studentGPAInformation) =>
          SELECT
            student.id                    `person_id`,
            a.periodBased.*
          FROM ethosStudentGPAReader
          lateral view explode(periodBased) a as periodBased

## Using the ROW_NUMBER function assigns a unique, sequential number to each row, starting with one,
  # according to the ordering of rows within the partition from the information pulled in the previous statement for the gpa.person_id, gpa.academicPeriod.id
  ## The PARTITION BY is dividing the rows into small partitions by gpa.person_id, gpa.academicPeriod.id preference field
  - statement: |
      (periodBasedStudentGPAinfo) =>
          SELECT
            gpa.person_id,
            gpa.value,            
            gpa.attemptedCredits,
            gpa.earnedCredits,
            gpa.qualityPoints,
            gpa.academicPeriod.id                          `academicPeriod_id`,           
            ROW_NUMBER() OVER (PARTITION BY gpa.person_id, gpa.academicPeriod.id ORDER BY attemptedCredits DESC) AS Row

FROM studentGPAInformation gpa
          INNER JOIN sfhedCourseOfferingReader off ON gpa.academicPeriod.id = off.`hed__Term__r.External_GUID__c`

#- statement: print periodBasedStudentGPAinfo

- statement: REFRESH sfhedProgramEnrollmentReader
  - statement: (sfhedProgramEnrollmentReaderCount) => SELECT COUNT(*) FROM sfhedProgramEnrollmentReader
  #- statement: print sfhedProgramEnrollmentReaderCount

- statement: |
      (courseEnrollmentInformation) =>
          SELECT
            s.hed__Status__c,
            s.hed__Contact__c,
            s.hed__Course_Offering__c,

pe.Id                                             `hed__Program_Enrollment__c`,
            pe.hed__Account__c                                `hed__Account__c`,

gpa.attemptedCredits                              `hed__Credits_Attempted__c`,
            gpa.earnedCredits                                 `hed__Credits_Earned__c`

FROM sectionInformation s
          LEFT JOIN distinctProgramEnrollent pe ON s.hed__Contact__c = pe.hed__Contact__c AND pe.Row = 1
          LEFT JOIN periodBasedStudentGPAinfo gpa ON s.person_id = gpa.person_id AND gpa.Row = 1

#- statement: print courseEnrollmentInformation
  - statement: (courseEnrollmentInformationCount) => SELECT COUNT(*) FROM courseEnrollmentInformation
  #- statement: print courseEnrollmentInformationCount

- statement: REFRESH sfhedCourseEnrollmentReader
  - statement: (sfhedCourseEnrollmentReaderCount) => SELECT COUNT(*) FROM sfhedCourseEnrollmentReader
  #- statement: print sfhedCourseEnrollmentReaderCount

#### Start - Update hed__Course_Enrollment__c ####

- statement: |
      (CourseEnrollmentInfoToUpdate) =>
          SELECT
            sf.Id,
            c.hed__Status__c,
            c.hed__Program_Enrollment__c,
            c.hed__Account__c,
            c.hed__Credits_Attempted__c,
            c.hed__Credits_Earned__c

FROM courseEnrollmentinformation c
          INNER JOIN sfhedCourseEnrollmentReader sf ON c.hed__Contact__c = sf.hed__Contact__c
            AND c.hed__Course_Offering__c = sf.hed__Course_Offering__c

#- statement: print CourseEnrollmentInfoToUpdate
  - statement: (CourseEnrollmentInfoToUpdateCount) => SELECT COUNT(*) FROM CourseEnrollmentInfoToUpdate
  #- statement: print CourseEnrollmentInfoToUpdateCount  
  - statement: UPDATE CourseEnrollmentInfoToUpdate INTO sfCourseEnrollmentWriter

- statement: REFRESH sfRecordTypeReader

#### End - Update hed__Course_Enrollment__c ####

#### Start - Insert hed__Course_Enrollment__c ####

- statement: |
      (CourseEnrollmentInfoToInsert) =>
          SELECT
            c.*

FROM courseEnrollmentinformation c
          LEFT ANTI JOIN sfhedCourseEnrollmentReader sf ON c.hed__Contact__c = sf.hed__Contact__c
            AND c.hed__Course_Offering__c = sf.hed__Course_Offering__c

#            (SELECT Id FROM sfRecordTypeReader)             `RecordTypeId`

#- statement: print CourseEnrollmentInfoToInsert
  - statement: (CourseEnrollmentInfoToInsertCount) => SELECT COUNT(*) FROM CourseEnrollmentInfoToInsert
  #- statement: print CourseEnrollmentInfoToInsertCount  
#  - statement: INSERT CourseEnrollmentInfoToInsert INTO sfCourseEnrollmentWriter

#### End - Insert hed__Course_Enrollment__c ####

## LOG

- statement: |
        (noContactFound) => 
          SELECT   
            sec.*
          FROM ethosSectionRegistrationInfo sec
          LEFT ANTI JOIN sfContactReader sfContact ON sec.person_id = sfContact.External_GUID__c

#- statement: print noContactFound
  - statement: (dataCount) => SELECT COUNT(*) FROM noContactFound
  #- statement: print dataCount

- statement: |
        (noCourseOfferingFound) =>
          SELECT   
            sec.*
          FROM ethosSectionRegistrationInfo sec
          LEFT ANTI JOIN sfhedCourseOfferingReader sfOff ON sec.section_id = sfOff.External_GUID__c

#- statement: print noCourseOfferingFound
  - statement: (dataCount) => SELECT COUNT(*) FROM noCourseOfferingFound
  #- statement: print dataCount

# Add more statements to convert, join, aggregrate, transform, and integrate your data