Keywords
Workflow - some process defined by user, it sequence of one step or many steps that do something, usually its run by some Schedule. Job, Pipeline all related terms
Workflow Engine - a tool, Django app created by Finmars SCSA that helps to create and execute Workflow. This service is absolutely separated from Finmars Platform (backend) due to security and logical concerns.
Schedule - part of Workflow Engine, this service allows to execute Workflow on some regular schedule (with crontab) and user can define Payload to that Schedule as well
Payload - an input of your Workflow, normally its a JSON object (dictionary) that will be passed into Workflow Code. Its accessible via kwargs.get("payload", None) Payload itself is user-defined as its workflows
Definition - old keyword, but basically its a YAML/JSON document that describes Workflow itself (meta-file)
Workflow (version 1) - First Generation of Worfklows, its defined in worfklow.json. This types are still supported, but deprecated. Consider using Workflow (version 2)
Example of its structure (JSON)
{
"workflow":
{
"user_code":"com.finmars.standard-workflow:simple-autoimport",
"is_manager": false,
"tasks": ["com.finmars.standard-workflow:simple-autoimport.task"]
}
}Example of its structure (YAML)
workflow:
user_code: com.finmars.standard-workflow:collect-price
is_manager: false
tasks:
- com.finmars.standard-workflow:collect-price.task
Workflow (version 2) - Second Generation of Workflows, its defined in workflow.json. This structure is more complex and generated by Workflow Engine itself. But user still able to tune it manually.
Workflow Engine look at "version": property, so "version": "2", is for Workflow (version 2). If not defined it will be Workflow (version 1)
{
"version": "2",
"workflow": {
"name": "Nested Test",
"user_code": "com.finmars.local:nested",
"default_payload": {
"report_date": "2024-10-10"
},
"nodes": [
{
"id": "b33a5bc711c739b7",
"data": {
"node": {
"name": "step-1",
"type": "workflow",
"notes": "",
"user_code": "step-1"
},
"workflow": {
"name": "Download Data",
"user_code": "com.finmars.example-etl:download"
},
"source_code": "\ndef main(self, *args, **kwargs):\n self.log(\"Hello World\")\n\n return {\"status\": \"success\"}\n"
},
"name": "step-1",
"label": "",
"width": 422,
"height": 598,
"inputs": {
"in": {
"id": "3f1c9322cca914ed",
"label": "Input",
"socket": {
"name": "socket"
},
"control": null,
"showControl": true
},
"payload_input": {
"id": "caf1ac1948c12178",
"label": "Payload",
"socket": {
"name": "payload_socket"
},
"control": null,
"showControl": true
}
},
"outputs": {
"out": {
"id": "57eee5fd73fec7e5",
"label": "Output",
"socket": {
"name": "socket"
},
"multipleConnections": true
}
},
"controls": {},
"position": {
"x": -361.12865172833614,
"y": -374.3578764111789
}
},
{
"id": "b5744926f4976461",
"data": {
"node": {
"name": "conditional-test",
"type": "workflow",
"notes": "",
"user_code": "conditional-test"
},
"workflow": {
"name": "Test Vault",
"user_code": "com.finmars.local:test-vault"
},
"source_code": "\ndef main(self, *args, **kwargs):\n self.log(\"Hello World\")\n\n return {\"status\": \"success\"}\n"
},
"name": "conditional-test",
"label": "",
"width": 417,
"height": 719,
"inputs": {
"in": {
"id": "5cee94f3086e8e3e",
"label": "Input",
"socket": {
"name": "socket"
},
"control": null,
"showControl": true
},
"payload_input": {
"id": "1e9f1cdb08f3b2e2",
"label": "Payload",
"socket": {
"name": "payload_socket"
},
"control": null,
"showControl": true
}
},
"outputs": {
"out": {
"id": "7ea2cdd214c5ba7e",
"label": "Output",
"socket": {
"name": "socket"
},
"multipleConnections": true
}
},
"controls": {},
"position": {
"x": 423.5227311216107,
"y": -488.5574374671881
}
},
{
"id": "4324a1babd889f5f",
"data": {
"node": {
"name": "final-step",
"type": "source_code",
"notes": "",
"user_code": "final-step"
},
"workflow": {
"name": "Test Vault",
"user_code": "com.finmars.local:test-vault"
},
"source_code": "\ndef main(self, *args, **kwargs):\n self.log(\"Hello World\")\n\n return {\"status\": \"success\"}\n"
},
"name": "final-step",
"label": "",
"width": 440,
"height": 790,
"inputs": {
"in": {
"id": "969915a7f2d9fcf2",
"label": "Input",
"socket": {
"name": "socket"
},
"control": null,
"showControl": true
},
"payload_input": {
"id": "2858c38d5fc4a37b",
"label": "Payload",
"socket": {
"name": "payload_socket"
},
"control": null,
"showControl": true
}
},
"outputs": {
"out": {
"id": "694544a7e2761259",
"label": "Output",
"socket": {
"name": "socket"
},
"multipleConnections": true
}
},
"controls": {},
"position": {
"x": 1050.2581884865385,
"y": -536.2804491695036
}
},
{
"id": "5de8268354a95a58",
"data": {
"node": {
"name": "step-2-payload-generator",
"type": "source_code",
"notes": "",
"user_code": "step-2-payload-generator"
},
"workflow": {},
"source_code": "\ndef main(self, *args, **kwargs):\n self.log(\"Hello World\")\n \n payload = kwargs.get(\"payload\")\n \n # options = payload.get(\"options\")\n \n # if self.user_code in options['options_steps']:\n # return\n \n \n # execution continue\n return {\"hello\": \"world\"}\n"
},
"name": "step-2-payload-generator",
"label": "",
"width": 316,
"height": 696,
"inputs": {
"in": {
"id": "1d0df286dd91ce9c",
"label": "Input",
"socket": {
"name": "socket"
},
"control": null,
"showControl": true
},
"payload_input": {
"id": "f56cfc9b414c599d",
"label": "Payload",
"socket": {
"name": "payload_socket"
},
"control": null,
"showControl": true
}
},
"outputs": {
"out": {
"id": "4c9d99bde3f8f940",
"label": "Output",
"socket": {
"name": "socket"
},
"multipleConnections": true
}
},
"controls": {},
"position": {
"x": 1.1510722982846584,
"y": -1111.1812663772207
}
}
],
"connections": [
{
"sourceOutput": "out",
"targetInput": "in",
"id": "36b8b8ebc0bcd3d9",
"source": "b33a5bc711c739b7",
"target": "b5744926f4976461"
},
{
"sourceOutput": "out",
"targetInput": "in",
"id": "9560ed0b2b5092d3",
"source": "b5744926f4976461",
"target": "4324a1babd889f5f"
},
{
"sourceOutput": "out",
"targetInput": "payload_input",
"id": "763afddc95919939",
"source": "5de8268354a95a58",
"target": "b5744926f4976461"
}
],
"comments": []
}
}