Map and Merge Delimited Strings (Salesforce Mulitselect Picklists, etc.)

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - MAP AND MERGE DELIMITED STRINGS (SALESFORCE MULITSELECT PICKLISTS, ETC.)

# Recipe ID - 
# Recipe URL - https://app.lingk.io/a/10932/tf/17465

# Description - This recipe demonstrates how to map and merge comma delimited values across systems. 
              # Salesforce picklist values are a good example as they are a semi-colon (;) delimited string and many times the input is a comma (,) delimited string
              # The Lingk platform has functions like transform() that simplify these data processing scenarios.

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - Salesforce
# Connectors - 
# Data Flows - Single Direction
# Connection Type - Lingk Platform to API

# Data Models
## Salesforce Objects Used:
#1. Sections
#2. Programs

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

###### Start - JSON connectors ######

# Source data
-
    name: sourceData
    type: json
    properties:
      jsonObject: >
       [
         { "multiselect": "a,b,c", "id": "5003000000D8cuI" },
         { "multiselect": "a, C , d", "id": "5003000000D8cuJ" }
       ]

-
    name: targetPicklist
    type: json
    properties:
      jsonObject: >
       [
         { "value": "Bachelor;Champion", "id": "5003000000D8cuI" },
         { "value": "Bachelor;Degree", "id": "5003000000D8cuJ" },
       ]

-
    name: mappingData
    type: json
    properties:
      jsonObject: >
       [
         { "map": [
                {
                  "key": "a",
                  "value": "Admin"
                },
                {
                  "key": "b",
                  "value": "Bachelor"
                },
                {
                  "key": "c",
                  "value": "Champion"
                },
                {
                  "key": "d",
                  "value": "Degree"
                }
              ]
           }
       ]

###### End - JSON connectors ######

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

# DEBUG
  #- statement: print mappingData

# ENRICH SOURCE DATA WITH MAPPING DATA  
  - statement: |
        (mergeMapData) => 
            select *, (select map from mappingData limit 1) map
            from sourceData

#- statement: print mergeMapData

# CLEANSE source data, MAP source to target data , AND JOIN existing data 
# IN Javascript you would (1) split the string, (2) loop over the values, (3) map the values between source and target and (4) reconstruct the array/string
#
# IN Spark SQL
# 1. Split the string 
#     split(mergeMapData.multiselect, ',')
#
# 2. Loop over the values - transform is like a FOR loop
#     transform(
#         ... Array ..., 
#         (x, i) -> ... Process each value (x) 
#              )
# 3. Map the values between source and target
#     element_at(
#                  ... mapping values...,
#                  ...source key...
#               )
#
# 4. Reconstruct the array respecting existing target values and output a ; delimited string
#         array_join(
#            array_union(
#                 .. source array #1... ,
#                 .. existing target array #2...
#              ),
#          ';'
#          ) 
#

# array_join - is concatenating the elements of the given array (array_union) using the delimiter and an optional string to replace nulls. If no value is set for nullReplacement, 
                # any null value is filtered.
  # array_union - Returns an array of the elements in the union of array1 and array2, without duplicates.
  - statement: |
        (test) => Select
          mergeMapData.id,
          array_join(
            array_union(
                transform(
                  split(mergeMapData.multiselect, ','), 
                  (x, i) -> 
                    element_at(
                      map_from_entries(
                        mergeMapData.map
                      ),
                      trim(BOTH FROM lower(x))
                    )
                ),
                split(targetPicklist.value,';')
              ),
          ';'
          ) 
         data
        from mergeMapData
          inner join targetPicklist on mergeMapData.id = targetPicklist.id

#- statement: print test

# Add more statements to convert, join, aggregrate, transform, and integrate your data