# _____ _ _____ __ # | __ \ (_) |_ _| / _| # | |__) |___ ___ _ _ __ ___ | | _ __ | |_ ___ # | _ // _ \/ __| | '_ \ / _ \ | | | '_ \| _/ _ \ # | | \ \ __/ (__| | |_) | __/ _| |_| | | | || (_) | # |_| \_\___|\___|_| .__/ \___| |_____|_| |_|_| \___/ # | | # |_| # # Project Name - SFTP TASKS # Recipe URL - https://app.lingk.io/a/10932/tf/17561 # Description - This recipe reads a file from SFTP processes the data and then overwrites the file with new data # To run this recipe configure an SFTP connector in your environment # Industry - Higher Ed # Business Process - Graduate Reporting # Systems - SFTP # Connectors - SFTP # Data Flows - Single Direction # Connection Type - SFTP to SFTP # Data Models #1. # Add Recipe notes / Change log information here! # _____ _ # / ____| | | # | | ___ _ __ _ __ ___ ___| |_ ___ _ __ ___ # | | / _ \| '_ \| '_ \ / _ \/ __| __/ _ \| '__/ __| # | |___| (_) | | | | | | | __/ (__| || (_) | | \__ \ # \_____\___/|_| |_|_| |_|\___|\___|\__\___/|_| |___/ # # CONNECTORS specify what data will be pulled into the in-memory database during processing connectors: # Configure SFTP credentials in your Environment before running this recipe # SFTP Setup - https://help.lingk.io/en/articles/296-sftp-setup-guide ###### Start - SFTP Reader connectors ###### - name: courseReader type: sftpReader format: csvReader properties: path: /demojam/course.csv ###### End - SFTP Reader connectors ###### ###### Start - SFTP Writer connectors ###### - name: courseWriter type: sftpWriter format: csvWriter properties: path: /demojam/course.csv ###### End - SFTP Writer connectors ###### ###### Start - LocalFile Writer connectors ###### - name: localCourseWriter type: localFileWriter format: csvWriter properties: fileName: course.csv imports: - name: appendFileImporter type: localFileImporter format: raw properties: fileName: course.csv ###### End - LocalFile Writer connectors ###### # _______ _ # |__ __| | | # | | __ _ ___| | _____ # | |/ _` / __| |/ / __| # | | (_| \__ \ <\__ \ # |_|\__,_|___/_|\_\___/ # Batch task used for parameterized by statements/connectors tasks: - name: deleteFile type: sftpOperation function: deleteFile parameters: target: /demojam/course.csv.bak - name: copyFile type: sftpOperation function: copyFile parameters: source: /demojam/course.csv target: /demojam/course.csv.bak - name: moveFile type: sftpOperation function: moveFile parameters: # port: 32 # default is 22 # credentials: sftp source: /demojam/course.csv.bak target: /demojam/backup/course.csv.bak - name: directoryListTask type: sftpOperation function: directoryList parameters: dirPath: "/demojam" # filter: "*.json" # ______ _ # | ____| | | # | |__ ___ _ __ _ __ ___ __ _| |_ ___ # | __/ _ \| '__| '_ ` _ \ / _` | __/ __| # | | | (_) | | | | | | | | (_| | |_\__ \ # |_| \___/|_| |_| |_| |_|\__,_|\__|___/ # Configures the format of the file written to SFTP readFormats: - name: csvReader type: delimited properties: delimiter: ',' header: true writeFormats: - name: csvWriter type: delimited properties: quoteAllFields: true delimiter: ',' header: true # _____ _ _ _ # / ____| | | | | | # | (___ | |_ __ _| |_ ___ _ __ ___ ___ _ __ | |_ ___ # \___ \| __/ _` | __/ _ \ '_ ` _ \ / _ \ '_ \| __/ __| # ____) | || (_| | || __/ | | | | | __/ | | | |_\__ \ # |_____/ \__\__,_|\__\___|_| |_| |_|\___|_| |_|\__|___/ # STATEMENTS specify how the data should be processed while in memory statements: #******************************************************************** D I S C L A I M E R *********************************************************************************************** # * # Note that in an effort to keep recipes optimized for DPH (Data Processing Hours), print statements should be commented out after development has concluded for a recipe. * # For more information on DPH optimization, please visit the following help article - https://help.lingk.io/en/articles/212-minimizing-data-processing-hours-on-the-lingk-platform * # * #******************************************************************** D I S C L A I M E R *********************************************************************************************** ### DEBUG ### GET LAST RUN DATA #- statement: print courseReader # Write file to SFTP - statement: INSERT courseReader INTO localCourseWriter - statement: | execute task --name copyFile --result copyFileTaskResult #- statement: print copyFileTaskResult - statement: | execute task --name moveFile --result moveFileTaskResult #- statement: print moveFileTaskResult - statement: | execute task --name deleteFile --result deleteFileTaskResult #- statement: print deleteFileTaskResult - statement: | execute task --name directoryListTask --output sftpFilePathsListing --result taskResult #- statement: print taskResult #- statement: print sftpFilePathsListing # Add more statements to convert, join, aggregrate, transform, and integrate your data