[Discuss-gnuradio] Recurring memory leak problems with iterative decoding [ GNU Radio 3.7.11.1]

/* -*- c++ -*- */
/*
* Copyright 2019 <+YOU OR YOUR COMPANY+>.
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3, or (at your option)
* any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this software; see the file COPYING. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street,
* Boston, MA 02110-1301, USA.
*/

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <gnuradio/io_signature.h>
#include "decodeLDPC_impl.h"
#include <string>
#include <vector>
#include <algorithm>
#include <fstream>

#define N 8176
#define K 7154
#define P 1022
using namespace std;
namespace gr {
namespace ccsds {

decodeLDPC::sptr
decodeLDPC::make(std::string alist_file, int code_type,int iterations, float sigma, int update_sigma, int pack)
{
return gnuradio::get_initial_sptr
(new decodeLDPC_impl(alist_file,code_type,iterations, sigma, update_sigma,pack));
}

/*
* The private constructor
*/
decodeLDPC_impl::decodeLDPC_impl(std::string alist_file, int code_type,int iterations, float sigma, int update_sigma,int pack)
: gr::sync_block("decodeLDPC",
gr::io_signature::make(0, 0, 0),
gr::io_signature::make(0, 0, 0)),
d_iterations(iterations),d_sigma(sigma),d_update_sigma(update_sigma),
d_file(alist_file),d_code_type(code_type), d_pack(pack)
{
//Initializing input and output ports
d_in_port = pmt::mp("in");
message_port_register_in(d_in_port);

d_out_port = pmt::mp("out");
message_port_register_out(d_out_port);

//Setting callback
set_msg_handler(d_in_port, boost::bind(&decodeLDPC_impl::decode, this ,_1) );

//Packing bits into bytes
d_pack_8 = new blocks::kernel::pack_k_bits(8);

switch(code_type)
{
case 0:
ldpcDecode = &decodeLDPC_impl::decodeFullC2;
d_file = "/home/mbkitine/Dropbox/Lulea/GRC/DeepSpace/gr-ccsds/lib/fec/ldpc/alist/C2_Alist.a";
break;
case 1:
ldpcDecode = &decodeLDPC_impl::decodeExpC2;
d_file = "/home/mbkitine/Dropbox/Lulea/GRC/DeepSpace/gr-ccsds/lib/fec/ldpc/alist/C2_Alist.a";
break;
case 2:
ldpcDecode = &decodeLDPC_impl::decodeAR4JA;
punct = 512;
d_file = "/home/mbkitine/Dropbox/Lulea/GRC/DeepSpace/gr-ccsds/lib/fec/ldpc/alist/AR4JA_r12_k1024n.a";
break;
}

//LDPC codec object
ccsdsLDPC = new ldpc(d_file);

//Printing properties of the LDPC codec
std::cout << "LDPC Decoder Initialized .." << std::endl;
std::cout << "Code rate : " << ccsdsLDPC->getCodeRate() << std::endl;
std::cout << "Check nodes: " << ccsdsLDPC->getnumCheckNodes() << std::endl;
std::cout << "Variable nodes : " << ccsdsLDPC->getnumVarNodes() << std::endl;

set_output_multiple(N);
}
std::vector<unsigned char>
decodeLDPC_impl::decodeFullC2(std::vector<float> softBits)
{
std::vector<float> logapp = ccsdsLDPC->decode(softBits,d_iterations,d_sigma);
std::vector<unsigned char> decodedBits = slice(logapp);

return decodedBits;
}

std::vector<unsigned char>
decodeLDPC_impl::decodeExpC2(std::vector<float> softBits)
{
// Removing the last two LLRs
softBits.pop_back();
softBits.pop_back();

// Insert LLRs for the first 18 elements
std::vector<float> codeword(18,-1000.00);
codeword.insert(codeword.end(),softBits.begin(),softBits.end());

// LDPC Decoding
std::vector<float> logapp = ccsdsLDPC->decode(codeword,d_iterations,d_sigma);

// Slicing log-APP ratios into hard bits
std::vector<unsigned char> decodedBits = slice(logapp);

// Remove the prefixed zero bits
decodedBits.erase(decodedBits.begin(),decodedBits.begin() + 18);

return decodedBits;
}

std::vector<unsigned char>
decodeLDPC_impl::decodeAR4JA(std::vector<float> softBits)
{
for(int i = 0; i < punct; i++)
softBits.push_back(0.0);
std::vector<float> logapp = ccsdsLDPC->decode(softBits,d_iterations,d_sigma);
std::vector<unsigned char> decodedBits = slice(logapp);
return decodedBits;
}

std::vector<unsigned char>
decodeLDPC_impl::slice(std::vector<float> logapp)
{
std::vector<unsigned char> bits;
unsigned char bit;
for(int i = 0; i < logapp.size(); i++)
bits.push_back( (logapp[i] > 0) ? 0x01 : 0x00);
return bits;
}

void
decodeLDPC_impl::decode(pmt::pmt_t msg)
{
pmt::pmt_t meta(pmt::car(msg));
pmt::pmt_t bits(pmt::cdr(msg));

std::vector<float> softBits = pmt::f32vector_elements(bits);

// LDPC Decoding
std::vector<unsigned char> decodedBits = (*this.*ldpcDecode)(softBits);

if(d_pack)
{
uint8_t *decoded_u8 = (uint8_t *)malloc(sizeof(uint8_t)*decodedBits.size()/8);
for(int i=0; i<decodedBits.size()/8; i++)
{
decoded_u8[i] = 0;
for(int j=0; j<8; j++)
{
decoded_u8[i] |= decodedBits[i*8+j]?(0x80>>j):0;
}
}
// Publishing data
pmt::pmt_t pdu(pmt::cons(pmt::PMT_NIL,pmt::make_blob(decoded_u8,decodedBits.size()/8)));
message_port_pub(d_out_port, pdu);
free(decoded_u8);
}
else
{
// Publishing data
pmt::pmt_t pdu(pmt::cons(pmt::PMT_NIL,pmt::make_blob(decodedBits.data(),decodedBits.size())));
message_port_pub(d_out_port, pdu);
}
decodedBits.clear();
softBits.clear();

}

/*
* Our virtual destructor.
*/
decodeLDPC_impl::~decodeLDPC_impl()
{
}

int
decodeLDPC_impl::work(int noutput_items,
gr_vector_const_void_star &input_items,
gr_vector_void_star &output_items)
{
// const float *in = (const float *) input_items[0];
// //<+OTYPE+> *out = (<+OTYPE+> *) output_items[0];

// std::vector<double> softBits;
// for(int i = 0; i < N; i++)
// softBits.push_back(in[i]);

// LDPC Decoding
//ldpc decoderLDPC = ldpc(d_file);
//std::vector<unsigned char> decodedBits = (*this.*ldpcDecode)(softBits);
//std::vector<double> decodedBits = decoderLDPC.decode(softBits,d_iterations,d_sigma);
// Do <+signal processing+>
//softBits.clear();
// Tell runtime system how many output items we produced.
return noutput_items;
}

} /* namespace ccsds */
} /* namespace gr */

/* -*- c++ -*- */
/*
* Copyright 2019 <+YOU OR YOUR COMPANY+>.
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3, or (at your option)
* any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this software; see the file COPYING. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street,
* Boston, MA 02110-1301, USA.
*/

#ifndef INCLUDED_CCSDS_DECODELDPC_IMPL_H
#define INCLUDED_CCSDS_DECODELDPC_IMPL_H

#include <ccsds/decodeLDPC.h>
//#include "fec/ldpc/soft_decoder.h"
#include "fec/ldpc/checknode.h"
#include "fec/ldpc/variablenode.h"
#include "fec/ldpc/ldpc.h"
#include <cassert>
#include <gnuradio/blocks/pack_k_bits.h>
namespace gr {
namespace ccsds {

class decodeLDPC_impl : public decodeLDPC
{
private:
int d_iterations;
float d_sigma;
int d_update_sigma;
int d_code_type;
bool d_pack;
blocks::kernel::pack_k_bits *d_pack_8;
int punct;
string d_file;
pmt::pmt_t d_in_port;
pmt::pmt_t d_out_port;
int** H; //Parity matrix
//double *inputData;
ldpc *ccsdsLDPC;
std::vector<unsigned char> decodeFullC2(std::vector<float> softBits);
std::vector<unsigned char> decodeExpC2(std::vector<float> softBits);
std::vector<unsigned char> decodeAR4JA(std::vector<float> softBits);
std::vector<unsigned char> (decodeLDPC_impl::*ldpcDecode)(std::vector<float> input);
std::vector<unsigned char> slice(std::vector<float> logapp);

public:
decodeLDPC_impl(std::string alist_file, int code_type, int iterations, float sigma, int update_sigma,int pack);
~decodeLDPC_impl();
void decode(pmt::pmt_t msg);

// Where all the action really happens
int work(int noutput_items,
gr_vector_const_void_star &input_items,
gr_vector_void_star &output_items);
};

} // namespace ccsds
} // namespace gr

#endif /* INCLUDED_CCSDS_DECODELDPC_IMPL_H */

#ifndef LDPC_H
#define LDPC_H
#include <vector>
#include <algorithm>
#include <string>
#include <cassert>
#include "checknode.h"
#include "variablenode.h"
#include "alist.h"
using namespace std;
class ldpc
{
private:
std::vector<checkNode> checkNodes;
std::vector<variableNode> varNodes;
int numCheckNodes;
int numVarNodes;
float codeRate;
public:
ldpc(string filename);
void readAlist(string filename, std::vector< std::vector<int> > &m_list,std::vector< std::vector<int> > &n_list);
std::vector<float> decode(std::vector<float> softBits, int iterations, float sigma);
float getCodeRate() { return codeRate;}
int getnumCheckNodes() { return numCheckNodes;}
int getnumVarNodes() { return numVarNodes;}

};

#endif // LDPC_H
/*
* The Turbo codec class implementation
* -------------------------------------------
* This class implements CCSDS LDPC decoder as specified by the
* CCSDS TM SYNCHRONIZATION AND CHANNEL CODING blue book standard (August 2011).
* It can also work generically with any LDPC code specified by an alist file.
* Author : Moses Browne Mwakyanjala
* Date : March 18th, 2019
* Institue : Lulea University of Technology
* E-mail : moses.browne.mwakyanjala@ltu.se
*/

#include "ldpc.h"
#define DEBUG false
ldpc::ldpc(string filename)
{
//Reading alist file
std::vector< std::vector<int> > m_list, n_list;
readAlist(filename,m_list,n_list);
codeRate = (n_list.size()*1.0 - m_list.size()*1.0)/(n_list.size()*1.0);

//Initializing check nodes
for(unsigned int i = 0; i < m_list.size(); i++)
checkNodes.push_back(checkNode(m_list[i]));

// Initializing variable nodes
for(unsigned int i = 0; i < n_list.size(); i++)
{
varNodes.push_back(variableNode(n_list[i]));
}

// Deriving code properties
numCheckNodes = m_list.size();
numVarNodes = n_list.size();
}

void
ldpc::readAlist(string filename, std::vector< std::vector<int> > &m_list,std::vector< std::vector<int> > &n_list)
{
alist myAlist(filename.c_str());
m_list = myAlist.get_mlist();
n_list = myAlist.get_nlist();
}

std::vector<float>
ldpc::decode(std::vector<float> softBits, int iterations, float sigma)
{
// ##############################################
// Initializing variable nodes
for(unsigned int i = 0; i < softBits.size(); i++)
varNodes[i].setLx(2*softBits[i]/(sigma*sigma));

// ###############################################
for(int z = 0; z < iterations; z++)
{
// ##########################################
// Pass Lq from variable nodes to check nodes
for(unsigned int v = 0; v < varNodes.size();v++)
{
std::vector<int> indicies = varNodes[v].getCheckNodes();
std::vector<double> lq = varNodes[v].getLq();
assert(lq.size() == indicies.size());
for(unsigned int c = 0; c < indicies.size();c++)
checkNodes[indicies[c]].updateLq(lq[c]);
}

// ##########################################
// Updating Checknode Lrs
for(unsigned int i = 0; i < checkNodes.size(); i++)
{
checkNodes[i].updateLr();
}

// ##########################################
for(unsigned int c = 0; c < checkNodes.size();c++)
{
std::vector<int> indicies = checkNodes[c].getVarNodes();
std::vector<double> lr = checkNodes[c].getLr();
assert(lr.size() == indicies.size());
for(unsigned int r = 0; r < indicies.size();r++)
varNodes[indicies[r]].updateLr(lr[r]);
}

// #########################################
// Update Lqs
for(unsigned int i = 0; i < varNodes.size(); i++)
varNodes[i].updateLq();

// ##########################################
// Updating logAPP
for(unsigned int v = 0; v < varNodes.size(); v++)
varNodes[v].updatelogAPP();
}
//std::cout << "Size of varNodes : " << varNodes.size() << std::endl;
std::vector<float> logAPP;
for(unsigned int i = 0; i <(varNodes.size() - getnumCheckNodes()); i++)
{
logAPP.push_back(varNodes[i].getlogAPP());
}
if (DEBUG)
{
std::cout << "Size of checknode Lq vector : " << std::endl;
std::cout << "Size of checknode Lr vector : " << std::endl;
}
//std::cout << "Size of logAPP vector : " << logAPP.size() << std::endl;
return logAPP;
}

Hello everyone,

I have finished writing a C++ LDPC decoder for the standard CCSDS C2 (8160,7136) LDPC code. In order to avoid memory allocation issues, I have decided to use std::vector<> vectors throughout the program, as opposed to the c-style mallocs. I am able to run the program and simulate the BER on my laptop (8 Gig ram) without any problems. The simulation is able to run for hours without any issues. However, the code experiences severe memory leaks when imported to GNU Radio. 

The LDPC class files [ldpc.cc and ldpc.h] as well as the GNU Radio wrapper files [decodeLDPC_impl.h and decodeLDPC_impl.cc]   are attached to this email. The wrapper class constructor initializes the LDPC decoder variable by specifying an "alist" file. It also initializes the message handler function called "decode" which use the LDPC codec to carry out LDPC decoding. 

//Constructor

set_msg_handler(d_in_port, boost::bind(&decodeLDPC_impl::decode, this ,_1) );

ccsdsLDPC = new ldpc(d_file);

The decode message handler receives soft bits from the recoverCADUSoft block, which are float values as shown in the code below. The soft bits are decoded by a wrapper function "decodeLDPC::decode" which use the function ldpc->decode(softbits,iterations,sigma). Iterative decoding requires a number of iterations to successful converge. I use 20 iterations. 

//Decode message handler

void

    decodeLDPC_impl::decode(pmt::pmt_t msg)

    {

        pmt::pmt_t meta(pmt::car(msg));

        pmt::pmt_t bits(pmt::cdr(msg));

        std::vector<float> softBits = pmt::f32vector_elements(bits);

        // LDPC Decoding

        std::vector<unsigned char> decodedBits = (*this.*ldpcDecode)(softBits);

        if(d_pack)

        {

            uint8_t *decoded_u8 = (uint8_t *)malloc(sizeof(uint8_t)*decodedBits.size()/8);

            for(int i=0; i<decodedBits.size()/8; i++)

            {

                decoded_u8[i] = 0;

                for(int j=0; j<8; j++)

                {

                    decoded_u8[i] |= decodedBits[i*8+j]?(0x80>>j):0;

                }

            }

            // Publishing data

            pmt::pmt_t pdu(pmt::cons(pmt::PMT_NIL,pmt::make_blob(decoded_u8,decodedBits.size()/8)));

            message_port_pub(d_out_port, pdu);

            free(decoded_u8);

        }

        else

        {

        // Publishing data

        pmt::pmt_t pdu(pmt::cons(pmt::PMT_NIL,pmt::make_blob(decodedBits.data(),decodedBits.size())));

                    message_port_pub(d_out_port, pdu);

        }

        decodedBits.clear();

        softBits.clear();

    }

My question is what could possibly be the cause of the memory leak I experience? There are no memory leaks when the class is used outside GNU Radio. To add more confusion, I experienced the same situation when I was working with another iterative decoder for Turbo code (only 10 iterations). The code was able to run smoothly in a C++ application but experienced memory leaks in GNU Radio.

I also have one question regarding buffering in GNU Radio. Since iterative decoding with a large number of iterations and large block sizes takes time to complete, the input pmt data that is not consumed immediately will have to be stored somewhere. Is that the case? Could that be the reason for the memory leak?

Regards,
Moses.