Join Salesforce data for better Excel reporting or CSV data feeds

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name -  JOIN SALESFORCE DATA FOR BETTER EXCEL REPORTING OR CSV DATA FEEDS

# Recipe ID - 
# Recipe URL - https://app.lingk.io/a/10932/tf/17463

# Description - # This recipe joins Salesforce data for a better excel reporting or csv data feeds

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - Salesforce
# Connectors - 
# Data Flows - Single Direction
# Connection Type - Lingk Local File to API

# Data Models
## Salesforce Objects Used:
#1. Contact
#2. hed__Contact__c

## Local Files
#1. output.csv
#2. output.xlsx

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure  Salesforce credentials in your Environment before running this recipe
  # Local File Setup - https://help.lingk.io/en/articles/126-local-file-connector-reference
  # Salesforce Setup - https://help.lingk.io/en/articles/267-salesforce-connector-setup-guide

###### Start - Salesforce Reader connectors ######

- 
    name: contactReader
    type: salesforceReader
    properties: 
      useBearerToken: true
      isSandbox: false
      query: SELECT Id, Name, Email FROM Contact

- 
    name: affiliationReader
    type: salesforceReader
    properties: 
      useBearerToken: true
      isSandbox: false
      query: SELECT Id, Name, hed__Contact__c FROM hed__Affiliation__c

###### End - Salesforce Reader connectors ######

###### Start - Local Writer connectors ######

# local writer for CSV and XLS
-
    name: outputWriterCSV
    type: localFileWriter
    format: commaFormat
    properties:
      fileName: output.csv

-
    name: outputWriterXLSX
    type: localFileWriter
    format: excelFormat
    properties:
      fileName: output.xlsx

###### End - Local Writer connectors ######

#   ______                         _       
#  |  ____|                       | |      
#  | |__ ___  _ __ _ __ ___   __ _| |_ ___ 
#  |  __/ _ \| '__| '_ ` _ \ / _` | __/ __|
#  | | | (_) | |  | | | | | | (_| | |_\__ \
#  |_|  \___/|_|  |_| |_| |_|\__,_|\__|___/

# for reading and writing flat file data
writeFormats:

-
    name: commaFormat
    type: delimited
    properties:
      quoteAllFields: true
      delimiter: ','
      header: true

-
    name: excelFormat
    type: excel
    properties:
      worksheets: data
      useHeader: true

# FORMATS specify how files should be processed as they are being read or written to

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

# get data from the reader
- statement: (source1) => select * from contactReader

# print data
#- statement: print source1

# get data from the reader
- statement: (source2) => select * from affiliationReader

# print data
#- statement: print source2

# get data from the reader
- statement: |
    (combinedSet) => select * 
                  from source1 
                      inner join source2 on source1.Id = source2.hed__Contact__c

# print data
#- statement: print combinedSet

- statement: (combinedSetCount) => select count(*) from combinedSet
#- statement: print combinedSetCount

# aggregate the data by email address domain
# Splits str around occurrences that match regex and returns an array with a length of at most limit
- statement: |
    (combinedSet) => select  
                  split(Email, '[@]')[1]            domain,
                  count(split(Email, '[@]')[1])     count
                  from source1 
                      inner join source2 on source1.Id = source2.hed__Contact__c
                   where Email != ""
                   group by split(Email, '[@]')[1]

# print data
#- statement: print combinedSet

- statement: (combinedSetCount) => select count(*) from combinedSet
#- statement: print combinedSetCount

# operate on data (insert back to SFTP, Salesforce or the Data Warehouse)
- statement: insert combinedSet into outputWriterXLSX
- statement: insert combinedSet into outputWriterCSV
#- statement: insert combinedSet into ftpWriter
#- statement: insert data into sfWriter

# print row count
- statement: (readerCount) => select count(*) from contactReader
#- statement: print readerCount

# Add more statements to convert, join, aggregrate, transform, and integrate your data