Get Salesforce documents base64encoded string

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - Get Salesforce documents base64encoded string

# Recipe URL - https://app.lingk.io/a/10932/tf/18243

# Description - # This recipe uses our HTTP connector to retrieve Salesforce document data. It then writes that data out to a Lingk hosted local file in a base 64 Encoded JSON file.

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - 
# Connectors - HTTP
# Data Flows - Single Direction
# Connection Type - API

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure HTTP credentials in your Environment before running this recipe
  # HTTP Setup - https://help.lingk.io/en/articles/304-http-connector-setup-guide

###### Start: HTTP Connectors #######

-
    name: GetAccessToken
    type: httpReader
    delayedRead: true
    properties:
      url: "https://americannationaluniversity2--coastaldev.my.salesforce.com/services/oauth2/token"
      headersJson: >
        { 
          "Content-Type": "application/x-www-form-urlencoded"
        } 
      httpMethod: POST
      body: grant_type=password&client_id={{env.Client_ID}}&client_secret={{env.Client_Secret}}&username={{env.User_name}}&password={{env.Password}}

## Get content version
-
    name: GetContentVersionInfo
    type: httpReader
    delayedRead: true
    properties:
      url: "https://americannationaluniversity2--coastaldev.my.salesforce.com/services/data/v54.0/query?q=SELECT%20Id%2C%20PathOnClient%2C%20VersionData%20FROM%20ContentVersion"
      headersJson: >
        { 
          "Authorization": "Bearer {{executionContext.accessTokenInfo.Access_token}}"
        } 
      httpMethod: GET

## Get version data
-
    name: GetVersionDataInfo
    type: httpV2Reader
    parameterizedBy: VersionDataInfo
    delayedRead: true
    properties:
      paginationType: stream
      url: "https://americannationaluniversity2--coastaldev.my.salesforce.com{{var.VersionData}}"
      headerJson: >
        { 
          "Content-Type": "application/octet-stream",
          "Authorization": "Bearer {{executionContext.accessTokenInfo.Access_token}}"
        } 
      httpMethod: GET

###### End: HTTP Connectors #######

###### Start: LocalFile Connectors #######

-
    name: FileWriter
    type: localFileWriter
    format: json
    properties:
      fileName: base64_data.json

###### End: LocalFile Connectors #######

#   ______                         _       
#  |  ____|                       | |      
#  | |__ ___  _ __ _ __ ___   __ _| |_ ___ 
#  |  __/ _ \| '__| '_ ` _ \ / _` | __/ __|
#  | | | (_) | |  | | | | | | (_| | |_\__ \
#  |_|  \___/|_|  |_| |_| |_|\__,_|\__|___/

# FORMATS specify how files should be processed as they are being read or written to

readFormats:
-  
    name: json
    type: json

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

- statement: REFRESH GetAccessToken
#- statement: print GetAccessToken

- statement: (accessTokenInfo) => SELECT access_token AS Access_token FROM GetAccessToken
- statement: INSERT accessTokenInfo INTO executionContext

################################################################# Uncomment code below to debug/test ################################################################

## Replace the access token variable with a hardcoded value

# - statement: (accessTokenInfo) => SELECT 
#                                     {{env.Access_token}} AS Access_token 
# - statement: print accessTokenInfo
# - statement: INSERT accessTokenInfo INTO executionContext

#####################################################################################################################################################################

# Gets the file info once the token was read from the execution context and provided to the HTTP connector

- statement: REFRESH GetContentVersionInfo
#- statement: print GetContentVersionInfo

# The data is retrieved as array so 'explode' function is used to get each record as a new row and visualise it
- statement: (explodedArray) => SELECT explode(records) as records FROM GetContentVersionInfo
#- statement: print explodedArray

# Providing a file name it retrieves the column needed from the table previously generated
- statement: (VersionDataInfo) => SELECT records.VersionData FROM explodedArray WHERE records.PathOnClient = {{env.file_name}}
#- statement: print VersionDataInfo

# Retrieves the version data info and inserted into the local file base64encoded
- statement: REFRESH GetVersionDataInfo
- statement: (VersionDataInfo) => SELECT * FROM GetVersionDataInfo
#- statement: print VersionDataInfo

- statement: INSERT VersionDataInfo INTO FileWriter

# Add more statements to convert, join, aggregrate, transform, and integrate your data