// MemBrain C Code export library V06.00.00.00

/*
--- License information and liability disclaimer ---
------------------
The source code exported by MemBrain including all supplementary source code
provided as part of the MemBrain installation is subject to licensing according
to the terms indicated in the license agreement that is displayed when
installing the MemBrain software.

You can also download and view this license agreement on the MemBrain
homepage (http://www.membrain-nn.de).

The terms of the license agreement do especially prohibit the commercial use of
any of the MemBrain source code items for a commercial purpose except the
user or company that intends to use the code in a commercial way is a
commercially registered MemBrain user and the owner of a corresponding commercial
MemBrain license key.

The MemBrain program and all associated supplementary software including but not
limited to the MemBrain DLL and all MemBrain generated source code are distributed
"as is". No warranty of any kind is expressed or implied. You use it at your own risk.
Thomas Jetter will not be liable for data loss, damages and loss of profits or
properties or any other kind of damages or loss caused by use or misuse of this
software.

Copyright 2003 - 2019 Thomas Jetter
*/


#include "NeuralNetDef.h"
#include <math.h>
#include <float.h>
#ifdef NEED_FLOAT_RAND
#include "Random.h"
#endif


#ifdef NEED_OUT_BUFFER
/// The output array for all neurons in the net
static MB_FLOAT    sNeuronOutputs[MB_OUT_ARRAY_SIZE];
#endif

/// Variable data for every neuron in the net
static SNeuronData        sNeuronData[MB_NEURON_COUNT];

/// Sum up the input signal to the neuron
static MB_FLOAT CalculateInputSum(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount);

/// Multiply up the input signal to the neuron
static MB_FLOAT CalculateInputMul(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount);

/// Determine the input signal to the neuron as maximum of all incoming links
static MB_FLOAT CalculateInputMax(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount);

/// Determine the input signal to the neuron as average of all incoming links
static MB_FLOAT CalculateInputAvg(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount);

#ifdef NEED_RAND_INPUT_FUNC
/// Determine the input signal to the neuron by random input selection over all incoming links
static MB_FLOAT CalculateInputRand(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount);
#endif

/// Snap activation to 0 if already near to
static void SnapToZero(T_ACT_FLOAT* pAct);

#ifdef FIRE_THRESHOLDS
/// Get the decision if the neuron shall fire according to activation and fire thresholds
static BOOL GetFireThresholdJudgement(const T_ACT_FLOAT* pAct, const SNeuronParms* pParms);
/// Get a random fire decision based on probability according to activation and fire thresholds
static BOOL GetFireProbabilityResult(const T_ACT_FLOAT* pAct, const SNeuronParms* pParms);
#endif


#ifdef ACT_FUNC_LOGISTIC
/// Limit of function argument during propagation
static const MB_FLOAT MAX_ARGUMENT_PROPAGATE = 10000;
#else
#ifdef ACT_FUNC_TAN_H
static const MB_FLOAT MAX_ARGUMENT_PROPAGATE = 10000;
#endif
#endif


/// Activation that is regarded to be 0
const T_ACT_FLOAT ACT_ZERO_RANGE = (T_ACT_FLOAT)FLT_MIN;



#ifdef ACT_FUNC_LOGISTIC
/// Logistic propagation function
static void PropagateLogistic(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT exponent);
#endif

#ifdef ACT_FUNC_IDENTICAL
/// Identical propagation function
static void PropagateIdentical(T_ACT_FLOAT* pAct, MB_FLOAT inputSum);
#endif

#ifdef ACT_FUNC_IDENTICAL_0_1
/// Identical 0 to 1 propagation function
static void PropagateIdentical01(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT leakage);
#endif

#ifdef ACT_FUNC_TAN_H
/// Tan Hyp propagation function
static void PropagateTanHyp(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT parTanHyp);
#endif

#ifdef ACT_FUNC_BINARY
/// Binary propagation function
static void PropagateBinary(T_ACT_FLOAT* pAct, MB_FLOAT inputSum);
#endif

#ifdef ACT_FUNC_MIN_EUCLID_DIST
/// Minimum Euclidian Distance propagation function
static void PropagateMinEuclidDist(T_ACT_FLOAT* pAct, T_LINK_IDX inLinkIdx, T_IN_LINK_COUNT inLinkCount);
#endif

#ifdef ACT_FUNC_IDENTICAL_M1_1
/// Identical 0 to 1 propagation function
static void PropagateIdenticalM11(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT leakage);
#endif

#ifdef ACT_FUNC_RELU
/// Rectified Linear Unit propagation function
static void PropagateReLu(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT leakage);
#endif

#ifdef ACT_FUNC_SOFTPLUS
/// Softplus propagation function
static void PropagateSoftplus(T_ACT_FLOAT* pAct, MB_FLOAT inputSum);
#endif

#ifdef ACT_FUNC_BIN_DIFF
/// Binary differentiable propagation function
static void PropagateBinDiff(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT binDiffSlope, MB_FLOAT leakage);
#endif

#ifdef ACT_FUNC_SOFTMAX
/// Softmax propagation function
static void PropagateSoftmax(T_ACT_FLOAT* pAct, MB_FLOAT inputSum);
#endif




#ifdef LINK_LENGTHS
/// Increment the current output index to adjust to the next write position into output buffer
static void IncrementOutputIdx(const SNeuronParms* pParms, SNeuronData* pData);
#endif

#ifdef NORMALIZATION
/// Normalize the activation of a neuron with the given user defined activation.
static BOOL NormalizeActivation(T_NEURON_COUNT index, T_ACT_FLOAT userAct);
/// De-normalize the activation of a neuron into the user defined activation.
static BOOL DeNormalizeActivation(T_NEURON_COUNT index, T_ACT_FLOAT* pUserAct);
#endif




/// Reset the given neuron
void NeuronReset(T_NEURON_COUNT index)
{
    SNeuronData* pData = &sNeuronData[index];

#ifdef NEED_OUT_BUFFER
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    MB_FLOAT* pOut = &sNeuronOutputs[pParms->outIdx];
#ifdef LINK_LENGTHS
    T_OUT_IDX i;

    // clear output buffer portion occupied by the neuron
    for (i = 0; i < pParms->maxOutLinkLen; i++)
    {
        *(pOut++) = (MB_FLOAT)0.0;
    }
    // reset index into output buffer
    pData->outIdxCurrent = pParms->outIdx;
#else
    *pOut = (MB_FLOAT)0.0;
#endif
#endif


    // reset activation
    pData->activation = (T_ACT_FLOAT)0.0;


#ifdef FIRE_DELAY
    // set fire delay to minimum value of 1 (neuron is ready to fire immediately after propagation)
    pData->fireDelay = 1;
#endif

#if MB_IN_NEURON_COUNT != 0
#ifdef SUSTAIN_FACT
    // all input neurons will ignore the sustain factor during first think step (behave as if new data had just been applied to them)
    if (index < MB_IN_NEURON_COUNT)
    {
        pData->firstThinkStep = TRUE;
    }
#endif
#endif
}


/// Cause a neuron to propagate (perform its calculation update)
void NeuronPropagate(T_NEURON_COUNT index)
{
    const SNeuronParms* pParms = &NEURON_PARMS[index];
#if defined(ACT_FUNC_LOGISTIC) || defined(ACT_FUNC_TAN_H) || defined (ACT_FUNC_BIN_DIFF)
    MB_FLOAT actFuncParm;
#endif
    MB_FLOAT threshold;
    T_NEURON_COUNT i;
    MB_FLOAT inputSum;
#if defined(ACT_FUNC_IDENTICAL_0_1) || defined(ACT_FUNC_IDENTICAL_M1_1) || defined (ACT_FUNC_RELU) || defined(ACT_FUNC_BIN_DIFF)
    MB_FLOAT leakage;
#endif
    SNeuronData* pData = &sNeuronData[index];
    T_ACT_FLOAT* pAct = &pData->activation;
    T_LINK_IDX inLinkIdx;
    T_IN_LINK_COUNT inLinkCount;
#if MB_IN_NEURON_COUNT != 0
    BOOL isInputNeuron = index < MB_IN_NEURON_COUNT;
#else
    BOOL isInputNeuron = FALSE;
#endif

#ifdef SUSTAIN_FACT
    // Ignore first think step if new data has just been applied to input neurons.
    // Else multiply current activation by the sustain factor
    if ((isInputNeuron == TRUE) && pData->firstThinkStep == TRUE)
    {
        pData->firstThinkStep = FALSE;
    }
    else
    {
        *pAct = *pAct * pParms->sustainFact;
    }
#else
    if (isInputNeuron != TRUE)
    {
        // No special sustain factors are used. Handle input neurons like they had a sustain factor of 1
        // (i.e. do nothing, leave activation as is).
        // Handle all other neurons like they had a sustain factor of 0 (reset activation to 0).
        *pAct = (T_ACT_FLOAT)0.0;
    }
#endif

    // for all neurons except input neurons
    if (isInputNeuron != TRUE)
    {
#if MB_HID_NEURON_COUNT != 0
        // Gather some parameters
        if (index >= MB_IN_NEURON_COUNT + MB_HID_NEURON_COUNT)
#endif
        { // for output neurons
            i = index - MB_IN_NEURON_COUNT - MB_HID_NEURON_COUNT;
            inLinkIdx = NEURON_PARMS_OUTPUT[i].inLinkIdx;
            inLinkCount = NEURON_PARMS_OUTPUT[i].inLinkCount;
            threshold = NEURON_PARMS_OUTPUT[i].actThres;
#ifdef ACT_FUNC_PARAMETRIZED
            actFuncParm = NEURON_PARMS_OUTPUT[i].actFuncParm;
#endif
#ifdef LEAKAGE_PARAMETRIZED
            leakage = NEURON_PARMS_OUTPUT[i].leakageParm;
#endif
        }
#if MB_HID_NEURON_COUNT != 0
        else
        { // for hidden neurons
            i = index - MB_IN_NEURON_COUNT;
            inLinkIdx = NEURON_PARMS_HIDDEN[i].inLinkIdx;
            inLinkCount = NEURON_PARMS_HIDDEN[i].inLinkCount;
            threshold = NEURON_PARMS_HIDDEN[i].actThres;
#ifdef ACT_FUNC_PARAMETRIZED
            actFuncParm = NEURON_PARMS_HIDDEN[i].actFuncParm;
#endif
#ifdef LEAKAGE_PARAMETRIZED
            leakage = NEURON_PARMS_HIDDEN[i].leakageParm;
#endif
        }
#endif

        // if neuron has input links connected to it
        if (inLinkCount > 0)
        {
#ifdef NEED_INPUT_FUNC
            switch (pParms->inputFunc)
            {
            case MB_IF_SUM:
                inputSum = CalculateInputSum(inLinkIdx, inLinkCount);
                break;

            case MB_IF_MUL:
                inputSum = CalculateInputMul(inLinkIdx, inLinkCount);
                break;

            case MB_IF_MAX:
                inputSum = CalculateInputMax(inLinkIdx, inLinkCount);
                break;

            case MB_IF_AVG:
                inputSum = CalculateInputAvg(inLinkIdx, inLinkCount);
                break;

#ifdef NEED_RAND_INPUT_FUNC
            case MB_IF_RAND:
                inputSum = CalculateInputRand(inLinkIdx, inLinkCount);
                break;
#endif

            default:
                inputSum = 0.0f;
                break;
            }
#else
            {
                // calculate the input signal sum
                inputSum = CalculateInputSum(inLinkIdx, inLinkCount);
            }
#endif
            // subtract the neuron activation threshold
            inputSum -= threshold;

            // calculate new activation
            switch (pParms->actFunc)
            {
#ifdef ACT_FUNC_LOGISTIC
      case MB_AF_LOGISTIC:
#ifndef ACT_FUNC_PARAMETRIZED
          actFuncParm = MB_EXP_LOG;
#endif
          PropagateLogistic(pAct, inputSum, actFuncParm);
          break;
#endif

#ifdef ACT_FUNC_IDENTICAL
      case MB_AF_IDENTICAL:
          PropagateIdentical(pAct, inputSum);
          break;
#endif

#ifdef ACT_FUNC_IDENTICAL_0_1
      case MB_AF_IDENTICAL_0_1:
#ifndef LEAKAGE_PARAMETRIZED
          leakage = MB_LEAKAGE;
#endif
          PropagateIdentical01(pAct, inputSum, leakage);
          break;
#endif

#ifdef ACT_FUNC_TAN_H
      case MB_AF_TAN_H:
#ifndef ACT_FUNC_PARAMETRIZED
          actFuncParm = MB_PAR_TAN_HYP;
#endif
          PropagateTanHyp(pAct, inputSum, actFuncParm);
          break;
#endif

#ifdef ACT_FUNC_BINARY
      case MB_AF_BINARY:
          PropagateBinary(pAct, inputSum);
          break;
#endif

#ifdef ACT_FUNC_MIN_EUCLID_DIST
      case MB_AF_MIN_EUCLID_DIST:
          PropagateMinEuclidDist(pAct, inLinkIdx, inLinkCount);
          break;
#endif

#ifdef ACT_FUNC_IDENTICAL_M1_1
      case MB_AF_IDENTICAL_M1_1:
#ifndef LEAKAGE_PARAMETRIZED
          leakage = MB_LEAKAGE;
#endif
          PropagateIdenticalM11(pAct, inputSum, leakage);
          break;
#endif

#ifdef ACT_FUNC_RELU
      case MB_AF_RELU:
#ifndef LEAKAGE_PARAMETRIZED
          leakage = MB_LEAKAGE;
#endif
          PropagateReLu(pAct, inputSum, leakage);
          break;
#endif

#ifdef ACT_FUNC_SOFTPLUS
      case MB_AF_SOFTPLUS:
          PropagateSoftplus(pAct, inputSum);
          break;
#endif

#ifdef ACT_FUNC_BIN_DIFF
      case MB_AF_BIN_DIFF:
#ifndef ACT_FUNC_PARAMETRIZED
          actFuncParm = MB_BIN_DIFF_SLOPE;
#endif
#ifndef LEAKAGE_PARAMETRIZED
          leakage = MB_LEAKAGE;
#endif
          PropagateBinDiff(pAct, inputSum, actFuncParm, leakage);
          break;
#endif

#ifdef ACT_FUNC_SOFTMAX
      case MB_AF_SOFTMAX:
          PropagateSoftmax(pAct, inputSum);
          break;
#endif

      default:
          *pAct = 0;
            }
        }
    }

    // snap small values to 0
    SnapToZero(pAct);
}


/// Cause a neuron to fire (calculate its output)
void NeuronFire(T_NEURON_COUNT index)
{
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    SNeuronData* pData = &sNeuronData[index];
    T_ACT_FLOAT* pAct = &pData->activation;
#ifdef NEED_OUT_BUFFER
    MB_FLOAT out;
    BOOL fire = TRUE;
#endif
#ifdef FIRE_DELAY
    // Check if neuron is subject to a fire delay time
    pData->fireDelay--;
    if (pData->fireDelay == 0)
    { // Neuron may fire immediately. Reload the delay time
        pData->fireDelay = pParms->recovTime;
    }
    else
    { // Neuron is not allowed to fire now
        fire = FALSE;
    }
#endif

    // handling the output separately from the activation is only makes sense if an output buffer is maintained.
#ifdef NEED_OUT_BUFFER

#ifdef FIRE_THRESHOLDS
    // Check if the fire thresholds allow for firing (eventually invokes random decision)
    if (fire == TRUE)
    {
        fire = GetFireThresholdJudgement(pAct, pParms);
    }
#endif

    if (fire)
    { // neuron shall fire
#ifdef OUT_FIRE_LEVEL_NOT_ACT
        if (pParms->fireLevel == MB_OFL_1)
        { // fire with a constant level of 1
            out = (MB_FLOAT)1.0;
        }
        else
        { // fire with current activation value
            out = (MB_FLOAT)*pAct;
        }
#else
        // fire with current activation value (standard if 'output fire level = 1' option is not used)
        out = (MB_FLOAT)*pAct;
#endif
    }
    else
    { // neuron shall not fire (i.e. output a signal of 0)
        out = (MB_FLOAT)0.0;
    }

#ifdef LINK_LENGTHS
    // If there are links with length > 1 in the net, handle writing to the output FIFO of the neuron now.
    if (pParms->maxOutLinkLen > 1)
    {
        IncrementOutputIdx(pParms, pData);
    }
    sNeuronOutputs[pData->outIdxCurrent] = out;
#else
    // No link lengths > 1 in the net: Every net only has one output buffer cell allocated.
    sNeuronOutputs[pParms->outIdx] = out;
#endif

#endif // NEED_OUT_BUFFER
}



/// Get the latest output signal of the neuron
MB_FLOAT NeuronGetLatestOutput(T_NEURON_COUNT index)
{
#ifdef NEED_OUT_BUFFER
    // Handle retrieval of output value from output buffer
#ifdef LINK_LENGTHS
    SNeuronData* pData = &sNeuronData[index];
    return sNeuronOutputs[pData->outIdxCurrent];
#else
    return sNeuronOutputs[NEURON_PARMS[index].outIdx];
#endif
#else
    // Handle retrieval of output value from activation (no output buffer used)
#ifdef FIRE_DELAY
    // Firing might be blocked by recovery time. Return 0 if so
    SNeuronData* pData = sNeuronData[index];
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    // if fire delay time is at the reload value then the neuron has just fired, i.e. get output from activation
    if (pData->fireDelay == pParms->recovTime)
        return (MB_FLOAT)pData->activation;
    else
        return (MB_FLOAT)0.0;

#else
    // No output recovery time > 1 in the net. Thus output is always firing
    return (MB_FLOAT)sNeuronData[index].activation;
#endif
#endif
}


#ifdef LINK_LENGTHS
/// Get a past output signal of the neuron
MB_FLOAT NeuronGetPastOutput(T_NEURON_COUNT index, T_LINK_LEN past)
{
    // retrieve the output signal from the output FIFO and return it
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    SNeuronData* pData = &sNeuronData[index];

    INT32 idx = (INT32)pData->outIdxCurrent - past;

    if (idx < pParms->outIdx)
    {
        idx += pParms->maxOutLinkLen;
    }

    return sNeuronOutputs[idx];
}
#endif



/// Set the activation of a neuron. <index> can range from 0 to MB_NEURON_COUNT - 1
void NeuronSetActivation(T_NEURON_COUNT index, T_ACT_FLOAT act)
{
    SNeuronData* pData = &sNeuronData[index];
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    UINT8 actFunc;

#ifdef NORMALIZATION
    BOOL normalize = NormalizeActivation(index, act);

    if (normalize != TRUE)
#endif
    {
        pData->activation = act;
        SnapToZero(&pData->activation);
        actFunc = pParms->actFunc;

        if (pParms->actFunc != MB_AF_IDENTICAL)
        {
            if (pData->activation > 1)
            {
                pData->activation = 1;
            }
            else
            {
                switch (actFunc)
                {
                case MB_AF_LOGISTIC:
                case MB_AF_IDENTICAL_0_1:
                case MB_AF_BINARY:
                    if (pData->activation < 0)
                    {
                        pData->activation = 0;
                    }
                    break;

                case MB_AF_IDENTICAL_M1_1:
                case MB_AF_TAN_H:
                case MB_AF_MIN_EUCLID_DIST:
                default:
                    if (pData->activation < -1)
                    {
                        pData->activation = -1;
                    }
                }
            }
        }
    }
#ifdef SUSTAIN_FACT
    pData->firstThinkStep = TRUE;
#endif
}

/// Get the activation value of a neuron. <idx> can range from 0 to MB_NEURON_COUNT - 1
T_ACT_FLOAT NeuronGetActivation(T_NEURON_COUNT index)
{
    SNeuronData* pData = &sNeuronData[index];
#ifdef NORMALIZATION
    T_ACT_FLOAT act;
    BOOL normalize = DeNormalizeActivation(index, &act);

    if (normalize == TRUE)
    {
        return act;
    }
    else
#endif
    {
        return pData->activation;
    }
}


#ifdef NORMALIZATION
/// Normalize the activation of a neuron with the given user defined activation. <index> can range from 0 to MB_NEURON_COUNT - 1
static BOOL NormalizeActivation(T_NEURON_COUNT index, T_ACT_FLOAT userAct)
{
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    SNeuronData* pData = &sNeuronData[index];
    T_NEURON_COUNT idx = 0;
    BOOL normalize = FALSE;
    T_NORM_FLOAT normLo;
    T_NORM_FLOAT normHi;
    T_NORM_FLOAT actLo;
    T_NORM_FLOAT m;
    T_NORM_FLOAT c;
    UINT8 actFunc = pParms->actFunc;


    if (index < MB_IN_NEURON_COUNT)
    { // input neurons
#ifdef INPUT_NORMALIZATION
        idx = index;
        normalize = NEURON_PARMS_INPUT[idx].useNormalize;
        if (normalize == TRUE)
        {
            normLo = NEURON_PARMS_INPUT[idx].normalizeLow;
            normHi = NEURON_PARMS_INPUT[idx].normalizeHigh;
        }
#else
        return FALSE;
#endif
    }
    else if (index >= MB_IN_NEURON_COUNT + MB_HID_NEURON_COUNT)
    { // output neurons
#ifdef OUTPUT_NORMALIZATION
        idx = index - MB_IN_NEURON_COUNT - MB_HID_NEURON_COUNT;;
        normalize = NEURON_PARMS_OUTPUT[idx].useNormalize;
        if (normalize == TRUE)
        {
            normLo = NEURON_PARMS_OUTPUT[idx].normalizeLow;
            normHi = NEURON_PARMS_OUTPUT[idx].normalizeHigh;
        }
#else
        return FALSE;
#endif
    }

    if (normalize)
    {
        if ((actFunc == MB_AF_IDENTICAL) || (actFunc == MB_AF_TAN_H) ||
            (actFunc == MB_AF_MIN_EUCLID_DIST) || (actFunc == MB_AF_IDENTICAL_M1_1))
        {
            actLo = -1;
        }
        else
        {
            actLo = 0;
        }
        m = (T_NORM_FLOAT)(((T_ACT_FLOAT)1.0 - actLo) / (normHi - normLo));
        c = (T_NORM_FLOAT)(actLo - m * normLo);

        pData->activation = (m * userAct + c);
    }

    return normalize;
}
#endif


#ifdef NORMALIZATION
/// De-normalize the activation of a neuron into the user defined activation.
static BOOL DeNormalizeActivation(T_NEURON_COUNT index, T_ACT_FLOAT* pUserAct)
{
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    SNeuronData* pData = &sNeuronData[index];
    T_NEURON_COUNT idx = 0;
    BOOL normalize = FALSE;
    T_NORM_FLOAT normLo;
    T_NORM_FLOAT normHi;
    T_NORM_FLOAT actLo;
    T_NORM_FLOAT m;
    T_NORM_FLOAT c;
    UINT8 actFunc = pParms->actFunc;


    if (index < MB_IN_NEURON_COUNT)
    {
#ifdef INPUT_NORMALIZATION
        idx = index;
        normalize = NEURON_PARMS_INPUT[idx].useNormalize;
        if (normalize == TRUE)
        {
            normLo = NEURON_PARMS_INPUT[idx].normalizeLow;
            normHi = NEURON_PARMS_INPUT[idx].normalizeHigh;
        }
#else
        return FALSE;
#endif
    }
    else if (index >= MB_IN_NEURON_COUNT + MB_HID_NEURON_COUNT)
    {
#ifdef OUTPUT_NORMALIZATION
        idx = index - MB_IN_NEURON_COUNT - MB_HID_NEURON_COUNT;;
        normalize = NEURON_PARMS_OUTPUT[idx].useNormalize;
        if (normalize == TRUE)
        {
            normLo = NEURON_PARMS_OUTPUT[idx].normalizeLow;
            normHi = NEURON_PARMS_OUTPUT[idx].normalizeHigh;
        }
#else
        return FALSE;
#endif
    }

    if (normalize)
    {
        if ((actFunc == MB_AF_IDENTICAL) || (actFunc == MB_AF_TAN_H) ||
            (actFunc == MB_AF_MIN_EUCLID_DIST) || (actFunc == MB_AF_IDENTICAL_M1_1))
        {
            actLo = -1;
        }
        else
        {
            actLo = 0;
        }
        m = (T_NORM_FLOAT)(((T_ACT_FLOAT)1.0 - actLo) / (normHi - normLo));
        c = (T_NORM_FLOAT)(actLo - m * normLo);

        *pUserAct = (pData->activation - c) / m;
        if ((*pUserAct >= -ACT_ZERO_RANGE) && (*pUserAct <= ACT_ZERO_RANGE))
        {
            *pUserAct = 0;
        }
    }

    return normalize;
}
#endif



/// Sum up the inputs to the neuron
static MB_FLOAT CalculateInputSum(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount)
{
    T_LINK_IDX idx = inLinkIndex;
    MB_FLOAT sum = 0.0f;
    T_IN_LINK_COUNT i;

    // just sum up every link output signal of all connected input links
    for (i = 0; i < inLinkCount; i++)
    {
        sum += NeuralLinkGetOutput(idx++);
    }

    return sum;
}

/// Multiply up the inputs to the neuron
static MB_FLOAT CalculateInputMul(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount)
{
    T_LINK_IDX idx = inLinkIndex;
    MB_FLOAT mul = 1.0f;
    T_IN_LINK_COUNT i;

    // just sum up every link output signal of all connected input links
    for (i = 0; i < inLinkCount; i++)
    {
        mul *= NeuralLinkGetOutput(idx++);
    }

    return mul;
}

/// Determine the input signal to the neuron as maximum of all incoming links. Return 0.0 if no input links connected
static MB_FLOAT CalculateInputMax(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount)
{
    T_LINK_IDX idx = inLinkIndex;
    MB_FLOAT max = inLinkCount > 0 ? -FLT_MAX : 0.0f;
    T_IN_LINK_COUNT i;

    // just sum up every link output signal of all connected input links
    for (i = 0; i < inLinkCount; i++)
    {
        MB_FLOAT out = NeuralLinkGetOutput(idx++);
        if (out > max)
        {
            max = out;
        }
    }

    return max;
}

/// Determine the input signal to the neuron as average of all incoming links
static MB_FLOAT CalculateInputAvg(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount)
{
    MB_FLOAT in;
    
    if (inLinkCount > 0)
    {
        in = CalculateInputSum(inLinkIndex, inLinkCount);
        in /= (MB_FLOAT)inLinkCount;
    }
    else
    {
        in = 0.0f;
    }

    return in;
}

#ifdef NEED_RAND_INPUT_FUNC
/// Determine the input signal to the neuron by random input selection over all incoming links
static MB_FLOAT CalculateInputRand(T_LINK_IDX inLinkIndex, T_IN_LINK_COUNT inLinkCount)
{
    MB_FLOAT in;

    if (inLinkCount > 0)
    {
        T_LINK_IDX offset = RandomGetIntRand(inLinkCount - 1);
        in = NeuralLinkGetOutput(inLinkIndex + offset);
    }
    else
    {
        in = 0.0f;
    }

    return in;
}
#endif


#ifdef LINK_LENGTHS
/// Increment the current output index into the output FIFO to adjust to the next write position
static void IncrementOutputIdx(const SNeuronParms* pParms, SNeuronData* pData)
{
    INT32 idx = (INT32)pData->outIdxCurrent + 1;

    if (idx >= pParms->outIdx + pParms->maxOutLinkLen)
    {
        idx -= pParms->maxOutLinkLen;
    }

    pData->outIdxCurrent = (T_OUT_IDX)idx;
}
#endif


#ifdef ACT_FUNC_LOGISTIC
/// Logistic propagation function
static void PropagateLogistic(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT exponent)
{
    MB_FLOAT argument = -exponent * inputSum;

    if (argument > MAX_ARGUMENT_PROPAGATE)
    {
        argument = MAX_ARGUMENT_PROPAGATE;
    }
    else if (argument < -MAX_ARGUMENT_PROPAGATE)
    {
        argument = -MAX_ARGUMENT_PROPAGATE;
    }

    *pAct += (T_ACT_FLOAT)(1 / (1 + exp(argument)));
    if (*pAct > 1)
    {
        *pAct = (T_ACT_FLOAT)1.0;
    }
}
#endif

#ifdef ACT_FUNC_IDENTICAL
/// Identical propagation function
static void PropagateIdentical(T_ACT_FLOAT* pAct, MB_FLOAT inputSum)
{
    *pAct += inputSum;
}
#endif

#ifdef ACT_FUNC_IDENTICAL_0_1
/// Identical 0 to 1 propagation function
static void PropagateIdentical01(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT leakage)
{
    *pAct += inputSum;
    if (*pAct > 1)
    {
        *pAct = (T_ACT_FLOAT)1 + leakage * (inputSum - (T_ACT_FLOAT)1);
    }
    else if (*pAct < 0)
    {
        *pAct = leakage * inputSum;
    }
}
#endif

#ifdef ACT_FUNC_TAN_H
/// Tan Hyp propagation function
static void PropagateTanHyp(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT parTanHyp)
{
    MB_FLOAT argument = parTanHyp * inputSum;

    if (argument > MAX_ARGUMENT_PROPAGATE)
    {
        argument = MAX_ARGUMENT_PROPAGATE;
    }
    else if (argument < -MAX_ARGUMENT_PROPAGATE)
    {
        argument = -MAX_ARGUMENT_PROPAGATE;
    }

    *pAct += (T_ACT_FLOAT)tanh(argument);
    if (*pAct > 1)
    {
        *pAct = (T_ACT_FLOAT)1;
    }
    else if (*pAct < -1)
    {
        *pAct = (T_ACT_FLOAT)-1;
    }
}
#endif

#ifdef ACT_FUNC_BINARY
/// Binary propagation function
static void PropagateBinary(T_ACT_FLOAT* pAct, MB_FLOAT inputSum)
{
    if (inputSum > 0)
    {
        *pAct = (T_ACT_FLOAT)1;
    }
}
#endif


#ifdef ACT_FUNC_MIN_EUCLID_DIST
/// Minimum Euclidian Distance propagation function
static void PropagateMinEuclidDist(T_ACT_FLOAT* pAct, T_LINK_IDX inLinkIdx, T_IN_LINK_COUNT inLinkCount)
{
    MB_FLOAT input;
    MB_FLOAT weight;
    MB_FLOAT temp;
    MB_FLOAT dist = 0;

    T_LINK_IDX idx = inLinkIdx;
    T_IN_LINK_COUNT i;

    if (inLinkCount == 0)
    {
        *pAct = 0;
        return;
    }

    // calculate the length of the input vector
    for (i = 0; i < inLinkCount; i++)
    {
        weight = NEURAL_LINK_PARMS[idx].weight;
        input = NeuralLinkGetInNeuronOutput(idx);
        temp = input - weight;
        dist += temp * temp;
        idx++;
    }
    dist = (MB_FLOAT)sqrt(dist);

    // The distance can be maximum twice the maximum length of the input
    // vector (if the signal is exactly opposite direction of the
    // weight).
    // The maximum length of the input vector is sqrt(mInLinks.GetCount()).
    // Thus the following calculation arranges for dist to be between 0 and 2 so
    // that the activation is always between -1 and 1.
    dist /= (MB_FLOAT)sqrt((MB_FLOAT)inLinkCount);

    *pAct = (T_ACT_FLOAT)1.0 - dist;

    if (*pAct > 1)
    {
        *pAct = (T_ACT_FLOAT)1;
    }
    else if (*pAct < -1)
    {
        *pAct = (T_ACT_FLOAT)-1;
    }
}
#endif

#ifdef ACT_FUNC_IDENTICAL_M1_1
/// Identical propagation function -1 to 1
static void PropagateIdenticalM11(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT leakage)
{
    *pAct += inputSum;
    if (*pAct > 1)
    {
        *pAct = (T_ACT_FLOAT)1 + leakage * (inputSum - (T_ACT_FLOAT)1);
    }
    else if (*pAct < -1)
    {
        *pAct = (T_ACT_FLOAT)-1 + leakage * (inputSum + (T_ACT_FLOAT)1);
    }
}
#endif

#ifdef ACT_FUNC_RELU
/// Rectified Linear Unit propagation function
static void PropagateReLu(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT leakage)
{
    if (inputSum > 0)
    {
        *pAct += inputSum;
    }
    else if (leakage > 0)
    {
        *pAct = inputSum * leakage;
    }
}
#endif

#ifdef ACT_FUNC_SOFTPLUS
/// Softplus propagation function
static void PropagateSoftplus(T_ACT_FLOAT* pAct, MB_FLOAT inputSum)
{
    *pAct += log((T_ACT_FLOAT)(1 + exp(inputSum)));
}
#endif

#ifdef ACT_FUNC_BIN_DIFF
/// Binary differentiable propagation function
static void PropagateBinDiff(T_ACT_FLOAT* pAct, MB_FLOAT inputSum, MB_FLOAT binDiffSlope, MB_FLOAT leakage)
{
    *pAct += inputSum * binDiffSlope;
    MB_FLOAT xMax = (MB_FLOAT)1 / binDiffSlope;

    if (*pAct > 1)
    {
        *pAct = (MB_FLOAT)1 + leakage * (inputSum - xMax);
    }
    else
    {
        *pAct = leakage * inputSum;
    }
}
#endif

#ifdef ACT_FUNC_SOFTMAX
/// Softmax activation function.
static void PropagateSoftmax(T_ACT_FLOAT* pAct, MB_FLOAT inputSum)
{
    *pAct += (T_ACT_FLOAT)exp(inputSum);
    // Postpropagation step is needed here after all softmax neuron activations of layer 
    // have been calculated!
}
#endif


/// Snap activation to 0 if already near to
static void SnapToZero(T_ACT_FLOAT* pAct)
{
    if ((*pAct >= -ACT_ZERO_RANGE) && (*pAct <= ACT_ZERO_RANGE))
        *pAct = 0;
}



#ifdef FIRE_THRESHOLDS
/// Get the decision if the neuron shall fire according to activation and fire thresholds
static BOOL GetFireThresholdJudgement(const T_ACT_FLOAT* pAct, const SNeuronParms* pParms)
{
    BOOL enable;

    // upper fire threshold reached or exceeded?
    if (*pAct >= pParms->fireThresHi)
    { // fire
        enable = TRUE;
    }
    else
    {
        // lower fire threshold exceeded?
        if (*pAct > pParms->fireThresLow)
        { // fire based on random decision
            enable = GetFireProbabilityResult(pAct, pParms);
        }
        else
        { // don't fire
            enable = FALSE;
        }
    }

    return enable;
}


/// Get a random fire decision based on probability according to activation and fire thresholds
static BOOL GetFireProbabilityResult(const T_ACT_FLOAT* pAct, const SNeuronParms* pParms)
{
    MB_FLOAT random01 = RandomGetRand01();
    MB_FLOAT fireThresHi = pParms->fireThresHi;
    MB_FLOAT fireThresLo = pParms->fireThresLow;

    if ((*pAct - fireThresLo) / (fireThresHi - fireThresLo) > random01)
    {
        return TRUE;
    }
    else
    {
        return FALSE;
    }
}
#endif

#ifdef MB_ACT_SUM_POST_PROC_LAYER_COUNT
void NeuronPostProcess(T_NEURON_COUNT index, T_ACT_FLOAT layerActSum)
{
    const SNeuronParms* pParms = &NEURON_PARMS[index];
    SNeuronData* pData = &sNeuronData[index];

    if (pParms->actFunc == MB_AF_SOFTMAX)
    { // Currently the only activation function which requires this
        pData->activation /= layerActSum;
    }
}
#endif