SFTP to Moodle (Upsert users)

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|                                
#

# Project Name - SFTP TO MOODLE (UPSERT USERS)

# Recipe URL - https://app.lingk.io/a/10932/tf/17565

# Description - This recipe will read a file from SFTP, transform the data and upsert the values into Moodle courses using the Lingk Connect Moodle plugin REST APIs.
              # To get started, configure Moodle and SFTP connectors in your environment  before running

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - SFTP, Moodle
# Connectors - SFTP, Moodle
# Data Flows - Single Direction
# Connection Type - SFTP to API

# Data Models
#1. Courses

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure SFTP credentials in your Environment before running this recipe
  # SFTP Setup - https://help.lingk.io/en/articles/296-sftp-setup-guide

###### Start - Moodle Reader connectors ######

# Read the users 
  # NOTE: this will read all the enrollments from the Moodle instance
-
    name: moodleCourseReader
    type: moodleReader
    properties:
      path: local/lingk/mvcswagger/gets.php?table=course

###### End - Moodle Reader connectors ######

###### Start - Moodle Writer connectors ######

# Create users
  # This should be paired with an INSERT ... INTO moodleCreateUserWriter
-
    name: moodleCreateCourseWriter
    type: moodleWriter
    properties:
      path: local/lingk/mvcswagger/creates.php?table=course

# Update users
  # This should be paired with an UPDATE ... INTO moodleCreateUserWriter
-
    name: moodleUpdateCourseWriter
    type: moodleWriter
    properties:
      path: local/lingk/mvcswagger/updates.php?table=course

###### End - Moodle Writer connectors ######

###### Start - SFTP Reader connectors ######

-
    name: courseReader
    type: sftpReader
    format: csv
    properties:
      path: /courses.csv

###### End - SFTP Reader connectors ######

#   ______                         _       
#  |  ____|                       | |      
#  | |__ ___  _ __ _ __ ___   __ _| |_ ___ 
#  |  __/ _ \| '__| '_ ` _ \ / _` | __/ __|
#  | | | (_) | |  | | | | | | (_| | |_\__ \
#  |_|  \___/|_|  |_| |_| |_|\__,_|\__|___/

# Configures the format of the file written to SFTP

readFormats:
-
    name: csv
    type: delimited
    properties:
      quoteAllFields: true
      delimiter: ','
      header: true

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

# Section: Debugging
# NOTE: this can be removed for production

- statement: (moodleData) => SELECT * FROM moodleCourseReader
  #- statement: print moodleData
  - statement: (moodleDataCount) => SELECT count(*) FROM moodleCourseReader
  #- statement: print moodleDataCount

- statement: (adapterData) => SELECT * FROM courseReader
  #- statement: print adapterData
  - statement: (adapterDataCount) => SELECT count(*) FROM courseReader
  #- statement: print adapterDataCount

# Section End: Debugging

# Section: Data retrieval
  - statement: >
        (allCourses) => 
          Select a.fullname,
                 a.shortname,
                 a.idnumber,
                 b.fullname   targetFullname,
                 b.shortname  targetShortname,
                 b.idnumber   targetIdNumber
            from courseReader a 
              left join moodleData b on a.idnumber = b.idnumber
  #- statement: print allCourses
  - statement: (allCoursesCount) => select count(*) from allCourses
  #- statement: print allCoursesCount

# Section: new courses
  - statement: >
        (newCourses) => 
          Select fullname,
                 shortname,
                 idnumber
            from allCourses 
            where targetIdNumber is null
  #- statement: print newCourses
  - statement: (newCoursesCount) => select count(*) from newCourses
  #- statement: print newCoursesCount

# Section: insert new courses
  - statement: insert newCourses into moodleCreateCourseWriter

# Section: existings courses to update
  - statement: >
        (existingCourses) => 
          Select fullname,
                 shortname,
                 idnumber
            from allCourses
            where targetIdNumber is not null
  #- statement: print existingCourses
  - statement: (existingCoursesCount) => select count(*) from existingCourses
  #- statement: print existingCoursesCount

# Section: update existing students 
  - statement: update existingCourses into moodleUpdateCourseWriter

# Add more statements to convert, join, aggregrate, transform, and integrate your data