JSON API to SOAP APIs and SOAP XML processing

#   _____           _              _____        __      
#  |  __ \         (_)            |_   _|      / _|     
#  | |__) |___  ___ _ _ __   ___    | |  _ __ | |_ ___  
#  |  _  // _ \/ __| | '_ \ / _ \   | | | '_ \|  _/ _ \ 
#  | | \ \  __/ (__| | |_) |  __/  _| |_| | | | || (_) |
#  |_|  \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ 
#                    | |                                
#                    |_|

# Project Name - JSON API TO SOAP APIS AND SOAP XML PROCESSING

# Recipe URL - https://app.lingk.io/a/10932/tf/17818

# Description - # This recipe dynamically populates values into an SOAP XML payload, submits the request and processes the result.
                # Jinja templates can be used to dynamically build the XML elements based on data from another connector (i.e. API request)
                # Credentials can be stored in a CSV in the local file connector and to dynamically populate credentials and make the recipe safe across environments.
                # To use this recipe, click Run!

# Industry - Higher Ed
# Business Process - Graduate Reporting

# Systems - Lingk
# Connectors - HTTP
# Data Flows - Single Direction
# Connection Type - API to SOAP

# Add Recipe notes / Change log information here!

#    _____                            _                 
#   / ____|                          | |                
#  | |     ___  _ __  _ __   ___  ___| |_ ___  _ __ ___ 
#  | |    / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __|
#  | |___| (_) | | | | | | |  __/ (__| || (_) | |  \__ \
#   \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_|  |___/
#

# CONNECTORS specify what data will be pulled into the in-memory database during processing

connectors:

# Configure HTTP credentials in your Environment before running this recipe
  # HTTP Setup - https://help.lingk.io/en/articles/304-http-connector-setup-guide

# Service documentation: http://www.dneonline.com/calculator.asmx

###### Start - JSON connectors ######

- name: numbersStatic
    type: json
    properties:
      jsonObject: |
          { "intA": 20, "intB": 30 }

###### End - JSON connectors ######

###### Start - HTTP Reader connectors ######

- name: soapWithStaticParams
    type: HTTPReader
    format: textFormatter
    parameterizedBy: numbersStatic # if the data is an array, it will submit the request one per row and join all the responses
    properties:
      url: http://www.dneonline.com/calculator.asmx
      httpMethod: POST
      acceptInvalidCerts: true
      body: |
        <?xml version="1.0" encoding="utf-8"?>
        <soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
          <soap:Body>
            <Add xmlns="http://tempuri.org/">
              <intA>{{var.intA}}</intA>
              <intB>{{var.intB}}</intB>
            </Add>
          </soap:Body>
        </soap:Envelope>
      headersJson: |
        {
          "Content-Type": "text/xml; charset=utf-8",
          "SOAPAction": "http://tempuri.org/Add"
        }
      #schemaRef: schemaName # optional from "

### GET NUMBER FROM AN HTTP SERVICE

- name: numbersHTTP
    type: httpReader
    format: jsonFormatter
    properties:
      httpMethod: GET
      url: https://postman-echo.com/get?intA=20&intB=30

- name: soapWithHTTPParams
    type: HTTPReader
    format: textFormatter # XML requires the text formatter
    parameterizedBy: numbersHTTP  # if the data is an array, it will submit the request one per row and join all the responses
    properties:
      url: http://www.dneonline.com/calculator.asmx
      httpMethod: POST
      acceptInvalidCerts: true
      body: |
        <?xml version="1.0" encoding="utf-8"?>
        <soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
          <soap:Body>
            <Add xmlns="http://tempuri.org/">
              <intA>{{var.args.intA}}</intA>
              <intB>{{var.args.intB}}</intB>
            </Add>
          </soap:Body>
        </soap:Envelope>
      headersJson: |
        {
          "Content-Type": "text/xml; charset=utf-8",
          "SOAPAction": "http://tempuri.org/Add"
        }
      #schemaRef: schemaName # optional from "

###### End - HTTP Reader connectors ######

#   ______                         _       
#  |  ____|                       | |      
#  | |__ ___  _ __ _ __ ___   __ _| |_ ___ 
#  |  __/ _ \| '__| '_ ` _ \ / _` | __/ __|
#  | | | (_) | |  | | | | | | (_| | |_\__ \
#  |_|  \___/|_|  |_| |_| |_|\__,_|\__|___/

readFormats:
  - name: jsonFormatter
    type: json

- name: textFormatter
    type: text
    # STATEMENTS specify how the file should be processed while in memory

#    _____ _        _                            _       
#   / ____| |      | |                          | |      
#  | (___ | |_ __ _| |_ ___ _ __ ___   ___ _ __ | |_ ___ 
#   \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __|
#   ____) | || (_| | ||  __/ | | | | |  __/ | | | |_\__ \
#  |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/

# STATEMENTS specify how the data should be processed while in memory

statements:

#******************************************************************** D I S C L A I M E R ***********************************************************************************************
#                                                                                                                                                                                       * 
# Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe.              *
# For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform      *
#                                                                                                                                                                                       * 
#******************************************************************** D I S C L A I M E R ***********************************************************************************************

# NOTE: You can add trasformations into your
  #- statement: print numbersStatic
  #- statement: print numbersHTTP

# this executes the connector when it has a parameterizedBy property
  - statement: refresh soapWithStaticParams

- statement: |
      (transformedData) => select 
        *   
      from soapWithStaticParams
  #- statement: print transformedData
  # Note: Spark SQL doesn't support namespaces, so you have to use local-name to traverse the nodes.
  - statement: |
      (selectedValues) => select 
        xpath_int( col,"//*[local-name()='Envelope']/*[local-name()='Body']/*[name()='AddResponse']/*[name()='AddResult']/text()") result
      from soapWithStaticParams
  #- statement: print selectedValues 
  #- statement: printschema selectedValues

# this executes the connector when it has a parameterizedBy property
  - statement: refresh soapWithHTTPParams

- statement: |
      (transformedData) => select 
        *   
      from soapWithHTTPParams
  #- statement: print transformedData
  # Note: Spark SQL doesn't support namespaces, so you have to use local-name to traverse the nodes.
  - statement: |
      (selectedValues) => select 
        xpath_int( col,"//*[local-name()='Envelope']/*[local-name()='Body']/*[name()='AddResponse']/*[name()='AddResult']/text()") result
      from soapWithHTTPParams
  #- statement: print selectedValues 
  #- statement: printschema selectedValues

# Add more statements to convert, join, aggregrate, transform, and integrate your data