Setting a page limit on the Ethos Connector

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - Setting a page limit on the Ethos Connector

# Recipe URL - https://app.lingk.io/a/10932/tf/18412

# Description -  # This recipe shows how you can set a page limit on the Ethos Reader Connector.
                 # Configure Ethos credentials in your Environment before running this recipe

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - 
# Connectors - Ethos
# Data Flows - Single Direction
# Connection Type - API

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure # Configure Ethos credentials in your Environment before running this recipe
  # Ethos Setup - https://help.lingk.io/en/articles/292-ellucian-ethos-connector-setup-guide

# This recipe does not write any data is used to demonstrate a simple selection of data

# Without limit count = 18
# with limit= 10 count = 18
-
    name: ethosAcademicLevelsAPIGet
    type: ethosReader
    delayedRead: true
    properties:
      path: /api/academic-levels
      dataModelVersion: 6
      debug: true
      #limit: 2
      #offset: 0

# Without limit count = 18
# with offset=0 and limit= 10 count = 28
-
    name: ethosAcademicLevelsAPIGet1
    type: ethosReader
    delayedRead: true
    properties:
      path: /api/academic-levels
      dataModelVersion: 6
      debug: true
      limit: 4
      offset: 0

# To test filter criteria
-
    name: ethosPersonsEmployee
    type: ethosReader
    delayedRead: true
    properties:
      path: /api/persons?criteria={"roles":[{"role":"employee"}]}&limit=10
      dataModelVersion: 12.3
      debug: true
      limit: 100
      offset: 0

-
    name: ethosPersonsVendor
    type: ethosReader
    delayedRead: true
    properties:
      path: /api/persons?criteria={"roles":[{"role":"vendor"}]}
      dataModelVersion: 12.3
      debug: true
      limit: 10
      offset: 0

-
    name: ethosPersonsFilterByEmail
    type: ethosReader
    delayedRead: true
    properties:
      path: /api/persons?criteria={"emails":[{"address":"newijsm@datatel.com"}]}
      dataModelVersion: 12.3
      debug: true

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

## Issue
  - statement: REFRESH ethosAcademicLevelsAPIGet
  - statement: print ethosAcademicLevelsAPIGet
  - statement: (dataCount) => select COUNT(*) from ethosAcademicLevelsAPIGet
  - statement: print dataCount

- statement: REFRESH ethosAcademicLevelsAPIGet1
  - statement: print ethosAcademicLevelsAPIGet1
  - statement: (dataCount1) => select COUNT(*) from ethosAcademicLevelsAPIGet1
  - statement: print dataCount1

## Filter criteria is working fine
  - statement: REFRESH ethosPersonsEmployee
  - statement: print ethosPersonsEmployee
  - statement: (dataCount2) => select COUNT(*) from ethosPersonsEmployee
  - statement: print dataCount2

- statement: REFRESH ethosPersonsVendor
  - statement: print ethosPersonsVendor
  - statement: (dataCount3) => select COUNT(*) from ethosPersonsVendor
  - statement: print dataCount3

- statement: REFRESH ethosPersonsFilterByEmail
  - statement: print ethosPersonsFilterByEmail
  - statement: (dataCount2) => select COUNT(*) from ethosPersonsFilterByEmail
  - statement: print dataCount2

# Add more statements to convert, join, aggregrate, transform, and integrate your data