Skip to content

Commit

Permalink
Added SecondChance algorithm to allow you to retry importing nodes th… (
Browse files Browse the repository at this point in the history 
#247)

* Added SecondChance algorithm to allow you to retry importing nodes that could not be imported because they did not find their parents. This may be the case if we have a node type of Object which has its own parent node type of Variable. For example, this could be a HasHistoricalConfiguration reference type. Also, the status about how many nodes we have imported shows numbers that include nodes that were not imported due to some error.

* Tests for checking the SecondChance algorithm.

---------

Co-authored-by: mkonnerth <matkonnerth@gmail.com>
  • Loading branch information
@xydan83 @matkonnerth
xydan83 and matkonnerth authored Dec 13, 2023
1 parent fd60349 commit 5e517fc
Show file tree
Hide file tree
Showing 7 changed files with 688 additions and 41 deletions.
168 changes: 128 additions & 40 deletions backends/open62541/src/import.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "conversion.h"
#include "NodesetLoader/NodesetLoader.h"
#include "RefServiceImpl.h"
#include "nodes/NodeContainer.h"

#include <assert.h>

Expand Down Expand Up @@ -115,7 +116,7 @@ static UA_NodeId getParentId(const NL_Node *node, UA_NodeId *parentRefId)
return parentId;
}

static void
static UA_StatusCode
handleObjectNode(const NL_ObjectNode *node, UA_NodeId *id,
const UA_NodeId *parentId, const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt, const UA_QualifiedName *qn,
Expand All @@ -134,13 +135,13 @@ handleObjectNode(const NL_ObjectNode *node, UA_NodeId *id,

// addNode_begin is used, otherwise all mandatory childs from type are
// instantiated
UA_Server_addNode_begin(server, UA_NODECLASS_OBJECT, *id, *parentId,
return UA_Server_addNode_begin(server, UA_NODECLASS_OBJECT, *id, *parentId,
*parentReferenceId, *qn, typeDefId, &oAttr,
&UA_TYPES[UA_TYPES_OBJECTATTRIBUTES],
node->extension, NULL);
}

static void
static UA_StatusCode
handleViewNode(const NL_ViewNode *node, UA_NodeId *id, const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt,
const UA_QualifiedName *qn, const UA_LocalizedText *description,
Expand All @@ -151,11 +152,11 @@ handleViewNode(const NL_ViewNode *node, UA_NodeId *id, const UA_NodeId *parentId
attr.description = *description;
attr.eventNotifier = (UA_Byte)atoi(node->eventNotifier);
attr.containsNoLoops = isValTrue(node->containsNoLoops);
UA_Server_addViewNode(server, *id, *parentId, *parentReferenceId, *qn, attr,
return UA_Server_addViewNode(server, *id, *parentId, *parentReferenceId, *qn, attr,
node->extension, NULL);
}

static void
static UA_StatusCode
handleMethodNode(const NL_MethodNode *node, UA_NodeId *id,
const UA_NodeId *parentId, const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt, const UA_QualifiedName *qn,
Expand All @@ -167,7 +168,7 @@ handleMethodNode(const NL_MethodNode *node, UA_NodeId *id,
attr.displayName = *lt;
attr.description = *description;

UA_Server_addMethodNode(server, *id, *parentId, *parentReferenceId, *qn,
return UA_Server_addMethodNode(server, *id, *parentId, *parentReferenceId, *qn,
attr, NULL, 0, NULL, 0, NULL, node->extension,
NULL);
}
Expand Down Expand Up @@ -198,7 +199,7 @@ static size_t getArrayDimensions(const char *s, UA_UInt32 **dims)
return arrSize;
}

static void handleVariableNode(const NL_VariableNode *node, UA_NodeId *id,
static UA_StatusCode handleVariableNode(const NL_VariableNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand Down Expand Up @@ -278,19 +279,20 @@ static void handleVariableNode(const NL_VariableNode *node, UA_NodeId *id,
}

//value is copied by open62541
UA_Server_addNode_begin(ServerContext_getServerObject(serverContext), UA_NODECLASS_VARIABLE, *id, *parentId,
UA_StatusCode Status = UA_Server_addNode_begin(ServerContext_getServerObject(serverContext), UA_NODECLASS_VARIABLE, *id, *parentId,
*parentReferenceId, *qn, typeDefId, &attr,
&UA_TYPES[UA_TYPES_VARIABLEATTRIBUTES],
node->extension, NULL);
//cannot call addNode finish, otherwise the nodes for e.g. range will be instantiated twice
//UA_Server_addNode_finish(server, *id);
UA_Variant_clear(&attr.value);

RawData_delete(data);
UA_free(attr.arrayDimensions);
return Status;
}

static void handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id,
static UA_StatusCode handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -303,11 +305,11 @@ static void handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id,
oAttr.isAbstract = isValTrue(node->isAbstract);
oAttr.description = *description;

UA_Server_addObjectTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
return UA_Server_addObjectTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
oAttr, node->extension, NULL);
}

static void handleReferenceTypeNode(const NL_ReferenceTypeNode *node,
static UA_StatusCode handleReferenceTypeNode(const NL_ReferenceTypeNode *node,
UA_NodeId *id, const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -322,11 +324,11 @@ static void handleReferenceTypeNode(const NL_ReferenceTypeNode *node,
attr.inverseName =
UA_LOCALIZEDTEXT(node->inverseName.locale, node->inverseName.text);

UA_Server_addReferenceTypeNode(server, *id, *parentId, *parentReferenceId,
return UA_Server_addReferenceTypeNode(server, *id, *parentId, *parentReferenceId,
*qn, attr, node->extension, NULL);
}

static void handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *id,
static UA_StatusCode handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -351,13 +353,13 @@ static void handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *i
}
}

UA_Server_addNode_begin(server, UA_NODECLASS_VARIABLETYPE, *id, *parentId,
return UA_Server_addNode_begin(server, UA_NODECLASS_VARIABLETYPE, *id, *parentId,
*parentReferenceId, *qn, UA_NODEID_NULL, &attr,
&UA_TYPES[UA_TYPES_VARIABLETYPEATTRIBUTES],
node->extension, NULL);
}

static void handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id,
static UA_StatusCode handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -370,11 +372,19 @@ static void handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id,
attr.description = *description;
attr.isAbstract = isValTrue(node->isAbstract);

UA_Server_addDataTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
return UA_Server_addDataTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
attr, node->extension, NULL);
}

static void addNodeImpl(ServerContext *serverContext, const NL_Node *node)
struct AddNodeContext
{
ServerContext* serverContext;
NodeContainer* problemNodes;
};

typedef struct AddNodeContext AddNodeContext;

static void addNodeImpl(AddNodeContext *context, NL_Node *node)
{
UA_NodeId id = node->id;
UA_NodeId parentReferenceId = UA_NODEID_NULL;
Expand All @@ -386,49 +396,56 @@ static void addNodeImpl(ServerContext *serverContext, const NL_Node *node)
UA_LocalizedText description =
UA_LOCALIZEDTEXT(node->description.locale, node->description.text);

UA_StatusCode addedNodeStatus;
switch (node->nodeClass)
{
case NODECLASS_OBJECT:
handleObjectNode((const NL_ObjectNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
addedNodeStatus = handleObjectNode((const NL_ObjectNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(context->serverContext));
break;

case NODECLASS_METHOD:
handleMethodNode((const NL_MethodNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
addedNodeStatus = handleMethodNode((const NL_MethodNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(context->serverContext));
break;

case NODECLASS_OBJECTTYPE:
handleObjectTypeNode((const NL_ObjectTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description,
ServerContext_getServerObject(serverContext));
addedNodeStatus = handleObjectTypeNode((const NL_ObjectTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description,
ServerContext_getServerObject(context->serverContext));
break;

case NODECLASS_REFERENCETYPE:
handleReferenceTypeNode((const NL_ReferenceTypeNode *)node, &id,
&parentId, &parentReferenceId, &lt, &qn,
&description, ServerContext_getServerObject(serverContext));
addedNodeStatus = handleReferenceTypeNode((const NL_ReferenceTypeNode *)node, &id,
&parentId, &parentReferenceId, &lt, &qn,
&description, ServerContext_getServerObject(context->serverContext));
break;

case NODECLASS_VARIABLETYPE:
handleVariableTypeNode((const NL_VariableTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description,
ServerContext_getServerObject(serverContext));
addedNodeStatus = handleVariableTypeNode((const NL_VariableTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description,
ServerContext_getServerObject(context->serverContext));
break;

case NODECLASS_VARIABLE:
handleVariableNode((const NL_VariableNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, serverContext);
addedNodeStatus = handleVariableNode((const NL_VariableNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, context->serverContext);
break;
case NODECLASS_DATATYPE:
handleDataTypeNode((const NL_DataTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
addedNodeStatus = handleDataTypeNode((const NL_DataTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(context->serverContext));
break;
case NODECLASS_VIEW:
handleViewNode((const NL_ViewNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
addedNodeStatus = handleViewNode((const NL_ViewNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(context->serverContext));
break;
}
// If a node was not added to the server due to an error, we add such a node
// to a special node container. We can then try to add such nodes later.
if(context->problemNodes != NULL && UA_StatusCode_isBad(addedNodeStatus))
{
NodeContainer_add(context->problemNodes, node);
}
}

unsigned short
Expand Down Expand Up @@ -541,28 +558,99 @@ static void addNonHierachicalRefs(UA_Server *server, NL_Node *node)
}
}

static size_t secondChanceAddNodes(ServerContext *serverContext,
NodeContainer **badStatusNodes,
const NodesetLoader_Logger *logger)
{
const size_t attemptsNum = 10;

size_t numberOfAllAddedNodes = 0;
// We will try to add all failed nodes in a certain number of attempts.
// This gives us the opportunity to add a chain like Object(child) ->
// Variable(parent).
// For example, this reference is used in the HasHistoricalConfiguration
// reference type.
size_t attempt = 1;
while (attempt != attemptsNum && (*badStatusNodes)->size != 0)
{
NodeContainer *local_badStatusNodes =
NodeContainer_new((*badStatusNodes)->size, false);
AddNodeContext context;
context.problemNodes = local_badStatusNodes;
context.serverContext = serverContext;
for (size_t counter = 0; counter < (*badStatusNodes)->size; counter++)
{
// Import to server again
addNodeImpl(&context, (*badStatusNodes)->nodes[counter]);
}
size_t counterOfAdddedNodesForOneAttempt =
(*badStatusNodes)->size - local_badStatusNodes->size;
numberOfAllAddedNodes += counterOfAdddedNodesForOneAttempt;
logger->log(logger->context, NODESETLOADER_LOGLEVEL_DEBUG,
"attempt (%zu), imported nodes: %zu", attempt,
counterOfAdddedNodesForOneAttempt);
NodeContainer_delete((*badStatusNodes));
(*badStatusNodes) = local_badStatusNodes;
attempt++;
}
return numberOfAllAddedNodes;
}

static void addNodes(NodesetLoader *loader, ServerContext *serverContext,
NodesetLoader_Logger *logger)
{
const NL_NodeClass order[NL_NODECLASS_COUNT] = {
NODECLASS_REFERENCETYPE, NODECLASS_DATATYPE, NODECLASS_OBJECTTYPE,
NODECLASS_VARIABLETYPE, NODECLASS_OBJECT, NODECLASS_METHOD,
NODECLASS_VARIABLE, NODECLASS_VIEW};

const size_t containerInitialSize = 100;

// If we have a problem adding nodes to the server, let's add references
// to these nodes to the container.
NodeContainer* badStatusNodes = NodeContainer_new(containerInitialSize, false);
// Since every cycle we add one node class we need to save
// counter of previous badStatusNodes because badStatusNodes
// will always be adding new bad nodes to one list, and we have to calculate
// the real number of bad status nodes on every single cycle.
size_t previous_loop_badStatusNodes_size = 0;

AddNodeContext context;
context.problemNodes = badStatusNodes;
context.serverContext = serverContext;
for (size_t i = 0; i < NL_NODECLASS_COUNT; i++)
{
const NL_NodeClass classToImport = order[i];
size_t cnt =
NodesetLoader_forEachNode(loader, classToImport, serverContext,
NodesetLoader_forEachNode(loader, classToImport, &context,
(NodesetLoader_forEachNode_Func)addNodeImpl);
if (classToImport == NODECLASS_DATATYPE)
{
importDataTypes(loader, ServerContext_getServerObject(serverContext));
}

// Now we can see the nodes that could not be added and can calculate
// and show the actual nodes added.
logger->log(logger->context, NODESETLOADER_LOGLEVEL_DEBUG,
"imported %ss: %zu", NL_NODECLASS_NAME[classToImport], cnt);
"imported %ss: %zu", NL_NODECLASS_NAME[classToImport],
cnt - (badStatusNodes->size - previous_loop_badStatusNodes_size));
previous_loop_badStatusNodes_size = badStatusNodes->size;
}

// second chance algorithm
if (badStatusNodes->size != 0)
{
logger->log(logger->context, NODESETLOADER_LOGLEVEL_WARNING,
"Couldn't import: %zu. Let's try adding non-imported "
"nodes a few more times.", badStatusNodes->size);
size_t numberOfAllAddedNodes =
secondChanceAddNodes(serverContext, &badStatusNodes, logger);
logger->log(logger->context, NODESETLOADER_LOGLEVEL_WARNING,
"imported after attempts: %zu", numberOfAllAddedNodes);
}

// Delete only reference and container. Not NL_Nodes objects.
NodeContainer_delete(badStatusNodes);

for (size_t i = 0; i < NL_NODECLASS_COUNT; i++)
{
const NL_NodeClass classToImport = order[i];
Expand Down
14 changes: 14 additions & 0 deletions backends/open62541/tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,20 @@ add_test(NAME conversion_Test
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMAND conversion)

add_executable(issue_246 issue_246.c)
target_include_directories(issue_246 PRIVATE ${CHECK_INCLUDE_DIR})
target_link_libraries(issue_246 PRIVATE NodesetLoader open62541::open62541 ${CHECK_LIBRARIES} ${PTHREAD_LIB})
add_test(NAME issue_246_Test
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMAND issue_246 ${CMAKE_CURRENT_SOURCE_DIR}/issue_246.xml)

add_executable(issue_246_2 issue_246_2.c)
target_include_directories(issue_246_2 PRIVATE ${CHECK_INCLUDE_DIR})
target_link_libraries(issue_246_2 PRIVATE NodesetLoader open62541::open62541 ${CHECK_LIBRARIES} ${PTHREAD_LIB})
add_test(NAME issue_246_2_Test
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMAND issue_246_2 ${CMAKE_CURRENT_SOURCE_DIR}/issue_246_2.xml)

if(${ENABLE_DATATYPEIMPORT_TEST})
add_subdirectory(dataTypeImport)
endif()
Expand Down
Loading

0 comments on commit 5e517fc

Please sign in to comment.