Replace unwanted characters from a string

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - REPLACE UNWANTED CHARACTERS FROM STRING

# Recipe ID - 
# Recipe URL - https://app.lingk.io/a/10932/tf/18132

# Description - # This recipe replaces unwanted characters from string data sets. This is often used to remove special characters from free form user driven data.

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - 
# Connectors - JSON
# Data Flows - Single Direction
# Connection Type - JSON

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure a JSON in your Environment before running this recipe
  # JSON Setup - https://help.lingk.io/en/articles/74-json-connector-reference

###### Start - JSON connectors ######

# This JSON contains static values that is used as the source data to be processed
-
    name: sourceLeadData
    type: json
    properties:
      jsonObject: >
       [
         { "leadId": "1", "firstName": "John", "lastName": "T!homas&)", "email": "mkyong123&@gmail.com"},
         { "leadId": "2", "firstName": "Anne@", "lastName": "Jacobs", "email": "mkyong123@gmail.com"}
       ]

-
    name: sourceLeadData1
    type: json
    properties:
      jsonObject: >
       [
         { "leadId": "1", "firstName": "John", "lastName": "Thomas", "leadScore": 88 },
         { "leadId": "2", "firstName": "Anne", "lastName": "Jacobs", "leadScore": 32 }
       ]

###### End - JSON connectors ######

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the file should be processed while in memory

statements:

# - statement: (sourceLeadEmailData) => SELECT email FROM sourceLeadData
  # - statement: print sourceLeadEmailData
  # - statement: stop
## reject invalid emails like -  mkyong123@gmail.a, mkyong123@.com, mkyong123@.com.com, mkyong@%*.com, mkyong..2002@gmail.com etc..
  - statement: |
      (emailValidation) =>
          SELECT DISTINCT
            if(email rlike ('[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,64}'), email, 'Not Valid')  `ValidEmail`
          FROM sourceLeadData

- statement: print emailValidation

# Remove the special characters from email address with the "replace" function 
  # The 'translate' function replaces the input on the 2nd parameter with an empty value - basically removing them
  - statement: |
       (replaceChar) =>
           SELECT
             replace(firstName, '@', '')                   `replaced`,
             translate(lastName, '!&)', '')                `translated`,
             translate(lastName, '!&)', '123')             `translated1`
           FROM sourceLeadData

- statement: print replaceChar

# Add more statements to convert, join, aggregrate, transform, and integrate your data