Commit 575789ee authored by David Nichols's avatar David Nichols
Browse files

refs #8 updated tests - made DB building blocks more generic

parent e9c5a93a
Pipeline #5569 passed with stage
in 6 minutes and 9 seconds
......@@ -19,6 +19,8 @@ hashdecl CreateOrderOptionInfo {
string workflow = UserApi::getConfigItemValue("create-workflow-name");
#! the optional workflow version
*string version = UserApi::getConfigItemValue("create-workflow-version");
#! mapper options
*hash<auto> mapper_opts;
}
#! creates a workflow order from the input data according to the configuration
......@@ -46,6 +48,47 @@ class BB_CreateOrder {
#! value indicating that duplicate orders should have a \c DUPLICATE-ORDER-KEY exception thrown
const RE_DUPLICATE = "DUPLICATE";
#! output data structure description for workflow order mappers
const WorkflowMapperOutput = {
"staticdata": {
"type": "hash",
"desc": "the initial static data for the order",
"mand": True,
},
"dynamicdata": {
"type": "hash",
"desc": "the initial dynamic data for the order",
},
"global_unique_key": {
"type": "hash",
"desc": "a hash giving one or more unique order keys for the order (across all workflows regardless of "
"workflowid, name, or version); keys are order key names and values are the string key values; if this "
"key already exists for any order in the system, then the order creation will fail with a "
"DUPLICATE-ORDER-KEY error; the hash key must be a valid order key, and the value is the unique key "
"value; this value will also be created as an order key",
},
"workflow_specific_unique_key": {
"type": "hash",
"desc": "a hash giving one or more unique order keys for the particular workflowid (which matches a "
"unique name and workflow version); keys are order key names and values are the string key values; if "
"any of the keys given already exists for an order with the target workflowid, then the order "
"creation will fail with a DUPLICATE-ORDER-KEY error; the hash key must be a valid order key, and the "
"value is the unique key value; this value will also be created as an order key",
},
"workflow_unique_key": {
"type": "hash",
"desc": "a hash giving one or more unique order keys for the particular workflow by name only (across all "
"workflows with the same name regardless of version); keys are order key names and values are the string "
"key values; if this key already exists for a workflow order with the same name, then the order creation "
"will fail with a DUPLICATE-ORDER-KEY error; the hash key must be a valid order key, and the value is the "
"unique key value; this value will also be created as an order key",
},
"orderkeys": {
"type": "hash",
"desc": "a hash of order keys for the order",
},
};
}
#! creates the order with the input data, returns the workflow_instanceid of the order created
......@@ -65,7 +108,7 @@ class BB_CreateOrder {
*/
static int createWorkflowOrder(hash<auto> order_context_data = {}, hash<CreateOrderOptionInfo> opts = <CreateOrderOptionInfo>{}, *reference<bool> dup) {
# create the workflow order with the mapper
Mapper mapper = UserApi::getMapper(opts.mapper);
Mapper mapper = UserApi::getMapper(opts.mapper, opts.mapper_opts);
hash<auto> order_data = mapper.mapData(order_context_data);
UserApi::logInfo("mapper %y mapped order data: %y", opts.mapper, order_data);
......
# name: BB_RegularExpressions
# version: 1.0
# desc: generic class for polling for files from an SFTP server
# desc: generic object defining constants for regular expression configuration item handling
# author: Qore Technologies, s.r.o.
%new-style
%require-types
%strict-args
%enable-all-warnings
const QRE_NONE = "NONE";
const QRE_IGNORE_CASE = "IGNORE-CASE";
const QRE_MULTILINE = "MULTILINE";
const QRE_IGNORE_CASE_MULTILINE = "IGNORE-CASE and MULTILINE";
class QRE {
public {
const NONE = "NONE";
const IGNORE_CASE = "IGNORE-CASE";
const MULTILINE = "MULTILINE";
const IGNORE_CASE_MULTILINE = "IGNORE-CASE and MULTILINE";
const QRE_MAP = {
QRE_NONE: 0,
QRE_IGNORE_CASE: RE_Caseless,
QRE_MULTILINE: RE_MultiLine,
QRE_IGNORE_CASE_MULTILINE: RE_Caseless | RE_MultiLine,
};
const MAP = {
NONE: 0,
IGNORE_CASE: RE_Caseless,
MULTILINE: RE_MultiLine,
IGNORE_CASE_MULTILINE: RE_Caseless | RE_MultiLine,
};
}
}
# END
\ No newline at end of file
......@@ -9,33 +9,52 @@
#! transfers data from a remote DB to a local DB
class BB_RemoteDb2LocalDbImportBase {
private {
#! prefix string for configuration items
/** to be used when multiple objects of this class are combined in one building block
*/
*string item_pfx;
}
#! constructor providing an optional item prefix
constructor(*string label) {
self.item_pfx = label;
}
#! performs the import action
import() {
*string item_pfx_str;
if (item_pfx) {
item_pfx_str = item_pfx + "-";
}
# get mapper name
string mapper_name = UserApi::getConfigItemValue("remote2local-db-mapper-name");
string mapper_name = UserApi::getConfigItemValue(item_pfx_str + "remote2local-db-mapper-name");
# get mapper for data conversions
Mapper mapper = UserApi::getMapper(mapper_name);
# get remote instance
string remote = UserApi::getConfigItemValue("remote2local-db-remote-instance");
string remote = UserApi::getConfigItemValue(item_pfx_str + "remote2local-db-remote-instance");
# get remote datasource name
string datasource = UserApi::getConfigItemValue("remote2local-db-remote-datasource");
string datasource = UserApi::getConfigItemValue(item_pfx_str + "remote2local-db-remote-datasource");
# get remote table name
string table = UserApi::getConfigItemValue("remote2local-db-remote-table");
string table = UserApi::getConfigItemValue(item_pfx_str + "remote2local-db-remote-table");
# get select stream options
hash<auto> opts;
# we use += to maintain the hash<auto> type
opts += UserApi::getConfigItemValue("remote2local-db-remote-options");
opts += UserApi::getConfigItemValue(item_pfx_str + "remote2local-db-remote-options");
*string column_name = UserApi::getConfigItemValue("remote2local-db-remote-select-column");
auto value = UserApi::getConfigItemValue("remote2local-db-remote-select-value-template");
*string column_name = UserApi::getConfigItemValue(item_pfx_str + "remote2local-db-remote-select-column");
auto value = UserApi::getConfigItemValue(item_pfx_str + "remote2local-db-remote-select-value-template");
if (exists column_name) {
opts."select"."where"{column_name} = value;
} else if (exists value) {
doWarning("CONFIG-ERROR", "the \"remote2local-db-remote-select-value-template\" config item value %y has been "
"ignored because config item \"remote2local-db-remote-select-column\" is not set", value);
doWarning("CONFIG-ERROR", "the \"%sremote2local-db-remote-select-value-template\" config item value %y "
"has been ignored because config item \"%sremote2local-db-remote-select-column\" is not set",
item_pfx_str, value, item_pfx_str);
}
UserApi::logInfo("opening select stream %s:%s:%s -> mapper %y", remote, datasource, table, mapper_name);
......@@ -79,42 +98,56 @@ class BB_RemoteDb2LocalDbImportBase {
}
#! returns the config items as documented in the class documentation
static hash<string, hash<ConfigItemInfo>> getConfigItems() {
hash<string, hash<ConfigItemInfo>> getConfigItems() {
*string item_pfx_str;
*string desc_pfx_str;
if (item_pfx) {
item_pfx_str = item_pfx + "-";
desc_pfx_str = item_pfx[0].upr() + item_pfx[1..] + ": ";
}
return {
# main configuration items
"remote2local-db-mapper-name": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-mapper-name": <ConfigItemInfo>{
"description": "the name of the mapper for the DB translations",
"config_group": "Remote DB Import",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import",
},
"remote2local-db-remote-instance": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-remote-instance": <ConfigItemInfo>{
"description": "the name of the remote qorus instance hosting the remote table",
"config_group": "Remote DB Import",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import",
},
"remote2local-db-remote-datasource": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-remote-datasource": <ConfigItemInfo>{
"description": "the name of the datasource in the remote instance",
"config_group": "Remote DB Import",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import",
},
"remote2local-db-remote-table": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-remote-table": <ConfigItemInfo>{
"description": "the name of the table in the remote datasource",
"config_group": "Remote DB Import",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import",
},
"remote2local-db-remote-options": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-remote-options": <ConfigItemInfo>{
"type": "hash",
"default_value": {},
"description": "options for the DbRemoteReceive object",
"config_group": "Remote DB Import",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import",
},
"remote2local-db-remote-select-column": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-remote-select-column": <ConfigItemInfo>{
"type": "*string",
"default_value": NOTHING,
"description": "the name of the column for the select criteria in the remote datasource",
"config_group": "Remote DB Import",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import",
},
"remote2local-db-remote-select-value-template": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-remote-select-value-template": <ConfigItemInfo>{
"type": "*string",
"default_value": NOTHING,
"description": "the column value for the select criteria in the remote table",
"config_group": "Remote DB Import",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import",
},
};
}
......
......@@ -10,6 +10,9 @@
#! transfers data from a remote DB to a local DB
class BB_RemoteDb2LocalDbImportWithRecovery inherits BB_RemoteDb2LocalDbImportBase {
constructor(*string label) : BB_RemoteDb2LocalDbImportBase(label) {
}
#! returns True if local data is present, False if not
bool checkLocal() {
# get mapper name
......@@ -34,16 +37,25 @@ class BB_RemoteDb2LocalDbImportWithRecovery inherits BB_RemoteDb2LocalDbImportBa
return False;
}
static private *hash<string, hash<ConfigItemInfo>> getConfigItems() {
private *hash<string, hash<ConfigItemInfo>> getConfigItems() {
*string item_pfx_str;
*string desc_pfx_str;
if (item_pfx) {
item_pfx_str = item_pfx + "-";
desc_pfx_str = item_pfx[0].upr() + item_pfx[1..] + ": ";
}
return BB_RemoteDb2LocalDbImportBase::getConfigItems() + {
# recovery items
"remote2local-db-recovery-column": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-recovery-column": <ConfigItemInfo>{
"description": "the name of the column for recovery",
"config_group": "Remote DB Import Recovery",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import Recovery",
},
"remote2local-db-recovery-value-template": <ConfigItemInfo>{
item_pfx_str + "remote2local-db-recovery-value-template": <ConfigItemInfo>{
"description": "value for recovery",
"config_group": "Remote DB Import Recovery",
"strictly_local": True,
"config_group": desc_pfx_str + "Remote DB Import Recovery",
},
};
}
......
......@@ -61,7 +61,7 @@ class BB_SftpPollerBase {
string mask = UserApi::getConfigItemValue("sftp-polling-mask");
if (UserApi::getConfigItemValue("sftp-polling-mask-regex")) {
poller_opts.regex_mask = mask;
poller_opts.reopts = QRE_MAP{UserApi::getConfigItemValue("sftp-polling-regex-opts")};
poller_opts.reopts = QRE::MAP{UserApi::getConfigItemValue("sftp-polling-regex-opts")};
} else {
poller_opts.mask = mask;
}
......@@ -252,8 +252,8 @@ class BB_SftpPollerBase {
},
"sftp-polling-regex-options": <ConfigItemInfo>{
"type": "string",
"default_value": QRE_NONE,
"allowed_values": keys QRE_MAP,
"default_value": QRE::NONE,
"allowed_values": keys QRE::MAP,
"description": "regular expression options for SFTP polling; only used if "
"\"sftp-polling-mask-regex\" is true",
"strictly_local": True,
......
......@@ -10,6 +10,9 @@
#! transfers data from a remote DB to a local DB
class BB_RemoteDb2LocalDbImportStep inherits QorusNormalStep, BB_RemoteDb2LocalDbImportWithRecovery {
constructor(*string label) : BB_RemoteDb2LocalDbImportWithRecovery(label) {
}
primary() {
import();
}
......
......@@ -13,25 +13,30 @@ class BB_TestDbRemote2LocalStep inherits BB_RemoteDb2LocalDbImportStep {
private {
#! default values for config items in this step
const DefaultConfigItemValues = {
"remote2local-db-mapper-name": "bb-test-db-step",
"remote2local-db-remote-datasource": "omquser",
"remote2local-db-remote-table": "bb_remote",
"remote2local-db-remote-select-column": "batch_id",
"remote2local-db-remote-select-value-template": "$static:batch_id",
"test-remote2local-db-mapper-name": "bb-test-db-step",
"test-remote2local-db-remote-datasource": "omquser",
"test-remote2local-db-remote-table": "bb_remote",
"test-remote2local-db-remote-select-column": "batch_id",
"test-remote2local-db-remote-select-value-template": "$static:batch_id",
# recovery items
"remote2local-db-recovery-column": "qorus_wfiid",
"remote2local-db-recovery-value-template": "$local:workflow_instanceid",
"test-remote2local-db-recovery-column": "qorus_wfiid",
"test-remote2local-db-recovery-value-template": "$local:workflow_instanceid",
};
}
constructor() : BB_RemoteDb2LocalDbImportStep("test") {
}
#! config items for this step
/** - rest-connection-name: the name of the billing system REST connection
*/
private *hash<string, hash<ConfigItemInfo>> getConfigItemsImpl() {
hash<string, hash<ConfigItemInfo>> rv = BB_RemoteDb2LocalDbImportStep::getConfigItemsImpl();
map rv{$1.key}.default_value = $1.value, DefaultConfigItemValues.pairIterator();
return rv;
return map {
$1.key: cast<hash<ConfigItemInfo>>($1.value + (
DefaultConfigItemValues{$1.key} ?
{"default_value": DefaultConfigItemValues{$1.key}}
: NOTHING
)),
}, BB_RemoteDb2LocalDbImportStep::getConfigItemsImpl().pairIterator();
}
}
# END
......@@ -28,7 +28,7 @@ class Test inherits QorusWorkflowTest {
);
const StepConfig = {
"remote2local-db-remote-instance": ConnectionName,
"test-remote2local-db-remote-instance": ConnectionName,
};
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment