static void amd_put_pages(struct sg_table *sg_head, void *client_context)
{
	int ret = 0;
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("put_pages: sg_head %p client_context: 0x%p\n",
			sg_head, client_context);
	MSG_DBG("put_pages: pid 0x%p, address 0x%llx, size:0x%llx\n",
			mem_context->pid,
			mem_context->va,
			mem_context->size);

	MSG_DBG("put_pages: mem_context->p2p_info %p\n",
				mem_context->p2p_info);

	if (ACCESS_ONCE(mem_context->free_callback_called)) {
		MSG_DBG("put_pages: free callback was called\n");
		return;
	}

	if (mem_context->p2p_info) {
		ret = rdma_interface->put_pages(&mem_context->p2p_info);
		mem_context->p2p_info = NULL;

		if (ret)
			MSG_ERR("put_pages failure: %d (callback status %d)",
					ret, mem_context->free_callback_called);
	} else
		MSG_ERR("put_pages: Pointer to p2p info is null\n");
}

/*
 * IOCTL operation; Return size for specified UMP memory.
 */
int ump_size_get_wrapper(u32 __user *argument, struct ump_session_data   *session_data)
{
	_ump_uk_size_get_s user_interaction;
	_mali_osk_errcode_t err;

	/* Sanity check input parameters */
	if (NULL == argument || NULL == session_data) {
		MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n"));
		return -ENOTTY;
	}

	if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) {
		MSG_ERR(("copy_from_user() in ump_ioctl_size_get()\n"));
		return -EFAULT;
	}

	user_interaction.ctx = (void *) session_data;
	err = _ump_ukk_size_get(&user_interaction);
	if (_MALI_OSK_ERR_OK != err) {
		MSG_ERR(("_ump_ukk_size_get() failed in ump_ioctl_size_get()\n"));
		return map_errcode(err);
	}

	user_interaction.ctx = NULL;

	if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) {
		MSG_ERR(("copy_to_user() failed in ump_ioctl_size_get()\n"));
		return -EFAULT;
	}

	return 0; /* success */
}

/*
 * IOCTL operation; Negotiate version of IOCTL API
 */
int ump_get_api_version_wrapper(u32 __user *argument, struct ump_session_data *session_data)
{
	_ump_uk_api_version_s version_info;
	_mali_osk_errcode_t err;

	/* Sanity check input parameters */
	if (NULL == argument || NULL == session_data) {
		MSG_ERR(("NULL parameter in ump_ioctl_get_api_version()\n"));
		return -ENOTTY;
	}

	/* Copy the user space memory to kernel space (so we safely can read it) */
	if (0 != copy_from_user(&version_info, argument, sizeof(version_info))) {
		MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n"));
		return -EFAULT;
	}

	version_info.ctx = (void *) session_data;
	err = _ump_uku_get_api_version(&version_info);
	if (_MALI_OSK_ERR_OK != err) {
		MSG_ERR(("_ump_uku_get_api_version() failed in ump_ioctl_get_api_version()\n"));
		return map_errcode(err);
	}

	version_info.ctx = NULL;

	/* Copy ouput data back to user space */
	if (0 != copy_to_user(argument, &version_info, sizeof(version_info))) {
		MSG_ERR(("copy_to_user() failed in ump_ioctl_get_api_version()\n"));
		return -EFAULT;
	}

	return 0; /* success */
}

int ump_unlock_wrapper(u32 __user *argument, struct ump_session_data   *session_data)
{
	_ump_uk_unlock_s user_interaction;

	/* Sanity check input parameters */
	if (NULL == argument || NULL == session_data) {
		MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n"));
		return -ENOTTY;
	}

	if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) {
		MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n"));
		return -EFAULT;
	}

	user_interaction.ctx = (void *) session_data;

	_ump_ukk_unlock(&user_interaction);

	user_interaction.ctx = NULL;

#if 0  /* No data to copy back */
	if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) {
		MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n"));
		return -EFAULT;
	}
#endif

	return 0; /* success */
}

/*
 * IOCTL operation; Do cache maintenance on specified UMP memory.
 */
int ump_msync_wrapper(u32 __user * argument, struct ump_session_data  * session_data)
{
	_ump_uk_msync_s user_interaction;

	/* Sanity check input parameters */
	if (NULL == argument || NULL == session_data)
	{
		MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n"));
		return -ENOTTY;
	}

	if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction)))
	{
		MSG_ERR(("copy_from_user() in ump_ioctl_msync()\n"));
		return -EFAULT;
	}

	user_interaction.ctx = (void *) session_data;

	_ump_ukk_msync( &user_interaction );

	user_interaction.ctx = NULL;

	if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction)))
	{
		MSG_ERR(("copy_to_user() failed in ump_ioctl_msync()\n"));
		return -EFAULT;
	}

	return 0; /* success */
}

/*
 * Read dpid's communication keys from its saved file.
 * Return value: 1 on success, -1 on error.
 */
static int Dpi_read_comm_keys(int *port)
{
   FILE *In;
   char *fname, *rcline = NULL, *tail;
   int i, ret = -1;

   fname = dStrconcat(dGethomedir(), "/.dillo/dpid_comm_keys", NULL);
   if ((In = fopen(fname, "r")) == NULL) {
      MSG_ERR("[Dpi_read_comm_keys] %s\n", dStrerror(errno));
   } else if ((rcline = dGetline(In)) == NULL) {
      MSG_ERR("[Dpi_read_comm_keys] empty file: %s\n", fname);
   } else {
      *port = strtol(rcline, &tail, 10);
      for (i = 0; *tail && isxdigit(tail[i+1]); ++i)
         SharedKey[i] = tail[i+1];
      SharedKey[i] = 0;
      ret = 1;
   }
   if (In)
      fclose(In);
   dFree(rcline);
   dFree(fname);

   return ret;
}

/*
 * IOCTL operation; Release reference to specified UMP memory.
 */
int ump_release_wrapper(u32 __user * argument, struct ump_session_data  * session_data)
{
	_ump_uk_release_s release_args;
	_mali_osk_errcode_t err;

	/* Sanity check input parameters */
	if (NULL == session_data)
	{
		MSG_ERR(("NULL parameter in ump_ioctl_release()\n"));
		return -ENOTTY;
	}

	/* Copy the user space memory to kernel space (so we safely can read it) */
	if (0 != copy_from_user(&release_args, argument, sizeof(release_args)))
	{
		MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n"));
		return -EFAULT;
	}

	release_args.ctx = (void*) session_data;
	err = _ump_ukk_release( &release_args );
	if( _MALI_OSK_ERR_OK != err )
	{
		MSG_ERR(("_ump_ukk_release() failed in ump_ioctl_release()\n"));
		return map_errcode(err);
	}


	return 0; /* success */
}

static int __init amd_peer_bridge_init(void)
{
	int result;

	MSG_INFO("init\n");


	result = amdkfd_query_rdma_interface(&rdma_interface);

	if (result < 0) {
		MSG_ERR("Can not get RDMA Interface (result = %d)\n", result);
		return result;
	}

	strcpy(amd_mem_client.name, AMD_PEER_BRIDGE_DRIVER_NAME);
	strcpy(amd_mem_client.version, AMD_PEER_BRIDGE_DRIVER_VERSION);
	ib_reg_handle = ib_register_peer_memory_client(&amd_mem_client,
						&ib_invalidate_callback);

	if (!ib_reg_handle) {
		MSG_ERR("Can not register peer memory client");
		return -EINVAL;
	}

	return 0;
}

/* Start a dpi filter plugin after accepting the pending connection
 * \Return
 * \li Child process ID on success
 * \li 0 on failure
 */
static int start_filter_plugin(struct dp dpi_attr)
{
   int newsock, old_stdout=-1, old_stdin=-1;
   socklen_t csz;
   struct sockaddr_un clnt_addr;
   pid_t pid;

   csz = (socklen_t) sizeof(clnt_addr);

   newsock = accept(dpi_attr.sock_fd, (struct sockaddr *) &clnt_addr, &csz);
   if (newsock == -1)
      ERRMSG("start_plugin", "accept", errno);

   dup2(STDIN_FILENO, old_stdin);
   if (dup2(newsock, STDIN_FILENO) == -1) {
      ERRMSG("start_plugin", "dup2", errno);
      MSG_ERR("ERROR in child proc for %s\n", dpi_attr.path);
      exit(1);
   }

   dup2(STDOUT_FILENO, old_stdout);
   if (dup2(newsock, STDOUT_FILENO) == -1) {
      ERRMSG("start_plugin", "dup2", errno);
      MSG_ERR("ERROR in child proc for %s\n", dpi_attr.path);
      exit(1);
   }
   if ((pid = fork()) == -1) {
      ERRMSG("main", "fork", errno);
      return 0;
   }
   if (pid == 0) {
      /* Child, start plugin */
      if (execl(dpi_attr.path, dpi_attr.path, (char*)NULL) == -1) {
         ERRMSG("start_plugin", "execl", errno);
         MSG_ERR("ERROR in child proc for %s\n", dpi_attr.path);
         exit(1);
      }
   }

   /* Parent, Close sockets fix stdio and return pid */
   if (dClose(newsock) == -1) {
      ERRMSG("start_plugin", "close", errno);
      MSG_ERR("ERROR in child proc for %s\n", dpi_attr.path);
      exit(1);
   }
   dClose(STDIN_FILENO);
   dClose(STDOUT_FILENO);
   dup2(old_stdin, STDIN_FILENO);
   dup2(old_stdout, STDOUT_FILENO);
   return pid;
}

/*
 * IOCTL operation; Allocate UMP memory
 */
int ump_allocate_wrapper(u32 __user * argument, struct ump_session_data  * session_data)
{
	_ump_uk_allocate_s user_interaction;
	_mali_osk_errcode_t err;

	/* Sanity check input parameters */
	if (NULL == argument || NULL == session_data)
	{
		MSG_ERR(("NULL parameter in ump_ioctl_allocate()\n"));
		return -ENOTTY;
	}

	/* Copy the user space memory to kernel space (so we safely can read it) */
	if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction)))
	{
		MSG_ERR(("copy_from_user() in ump_ioctl_allocate()\n"));
		return -EFAULT;
	}

	user_interaction.ctx = (void *) session_data;

	err = _ump_ukk_allocate( &user_interaction );
	if( _MALI_OSK_ERR_OK != err )
	{
		DBG_MSG(1, ("_ump_ukk_allocate() failed in ump_ioctl_allocate()\n"));
		return map_errcode(err);
	}
	user_interaction.ctx = NULL;

	if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction)))
	{
		/* If the copy fails then we should release the memory. We can use the IOCTL release to accomplish this */
		_ump_uk_release_s release_args;

		MSG_ERR(("copy_to_user() failed in ump_ioctl_allocate()\n"));

		release_args.ctx = (void *) session_data;
		release_args.secure_id = user_interaction.secure_id;

		err = _ump_ukk_release( &release_args );
		if(_MALI_OSK_ERR_OK != err)
		{
			MSG_ERR(("_ump_ukk_release() also failed when trying to release newly allocated memory in ump_ioctl_allocate()\n"));
		}

		return -EFAULT;
	}

	return 0; /* success */
}

/*! Scans a service directory in dpi_dir and fills dpi_attr
 * \Note
 * Caller must allocate memory for dpi_attr.
 * \Return
 * \li 0 on success
 * \li -1 on failure
 * \todo
 * Add other file types, but first we need to add files associated with a dpi
 * to the design.
 */
int get_dpi_attr(char *dpi_dir, char *service, struct dp *dpi_attr)
{
   char *service_dir = NULL;
   struct stat statinfo;
   enum file_type ftype;
   int ret = -1;
   DIR *dir_stream;
   struct dirent *dir_entry = NULL;

   service_dir = dStrconcat(dpi_dir, "/", service, NULL);
   if (stat(service_dir, &statinfo) == -1) {
      ERRMSG("get_dpi_attr", "stat", errno);
      MSG_ERR("file=%s\n", service_dir);
   } else if ((dir_stream = opendir(service_dir)) == NULL) {
      ERRMSG("get_dpi_attr", "opendir", errno);
   } else {
      /* Scan the directory looking for dpi files.
       * (currently there's only the dpi program, but in the future
       *  there may also be helper scripts.) */
      while ( (dir_entry = readdir(dir_stream)) != NULL) {
         if (dir_entry->d_name[0] == '.')
            continue;

         ftype = get_file_type(dir_entry->d_name);
         switch (ftype) {
            case DPI_FILE:
               dpi_attr->path =
                  dStrconcat(service_dir, "/", dir_entry->d_name, NULL);
               dpi_attr->id = dStrdup(service);
               dpi_attr->port = 0;
               dpi_attr->pid = 1;
               if (strstr(dpi_attr->path, ".filter") != NULL)
                  dpi_attr->filter = 1;
               else
                  dpi_attr->filter = 0;
               ret = 0;
               break;
            default:
               break;
         }
      }
      closedir(dir_stream);

      if (ret != 0)
         MSG_ERR("get_dpi_attr: No dpi plug-in in %s/%s\n",
                 dpi_dir, service);
   }
   dFree(service_dir);
   return ret;
}

static void block_allocator_release(void * ctx, ump_dd_mem * handle)
{
	block_allocator * allocator;
	block_info * block, * next;

	BUG_ON(!ctx);
	BUG_ON(!handle);

	allocator = (block_allocator*)ctx;
	block = (block_info*)handle->backend_info;
	BUG_ON(!block);

	if (down_interruptible(&allocator->mutex)) {
		MSG_ERR(("Allocator release: Failed to get mutex - memory leak\n"));
		return;
	}

	while (block) {
		next = block->next;

		BUG_ON( (block < allocator->all_blocks) || (block > (allocator->all_blocks + allocator->num_blocks)));

		block->next = allocator->first_free;
		allocator->first_free = block;
		allocator->num_free++;

		block = next;
	}
	DBG_MSG(3, ("%d blocks free after release call\n", allocator->num_free));
	up(&allocator->mutex);

	vfree(handle->block_array);
	handle->block_array = NULL;
}

/*
 * Read all the available data from a filedescriptor.
 * This is intended for short answers, i.e. when we know the server
 * will write it all before being preempted. For answers that may come
 * as an stream with delays, non-blocking is better.
 * Return value: read data, or NULL on error and no data.
 */
static char *Dpi_blocking_read(int fd)
{
   int st;
   const int buf_sz = 8*1024;
   char buf[buf_sz], *msg = NULL;
   Dstr *dstr = dStr_sized_new(buf_sz);

   do {
      st = read(fd, buf, buf_sz);
      if (st < 0) {
         if (errno == EINTR) {
            continue;
         } else {
            MSG_ERR("[Dpi_blocking_read] %s\n", dStrerror(errno));
            break;
         }
      } else if (st > 0) {
         dStr_append_l(dstr, buf, st);
      }
   } while (st == buf_sz);

   msg = (dstr->len > 0) ? dstr->str : NULL;
   dStr_free(dstr, (dstr->len > 0) ? FALSE : TRUE);
   return msg;
}

static unsigned long amd_get_page_size(void *client_context)
{
	unsigned long page_size;
	int result;
	struct amd_mem_context *mem_context =
		(struct amd_mem_context *)client_context;

	MSG_DBG("get_page_size: context: %p\n", client_context);
	MSG_DBG("get_page_size: pid 0x%p, address 0x%llx, size:0x%llx\n",
			mem_context->pid,
			mem_context->va,
			mem_context->size);


	result = rdma_interface->get_page_size(
				mem_context->va,
				mem_context->size,
				mem_context->pid,
				&page_size);

	if (result) {
		MSG_ERR("Could not get page size. %d", result);
		/* If we failed to get page size then do not know what to do.
		 * Let's return some default value
		 */
		return 4096;
	}

	return page_size;
}

/*! Establish handler for termination signals
 * and register cleanup with atexit */
void est_dpi_terminator()
{
   struct sigaction act;
   sigset_t block;

   sigemptyset(&block);
   sigaddset(&block, SIGHUP);
   sigaddset(&block, SIGINT);
   sigaddset(&block, SIGQUIT);
   sigaddset(&block, SIGTERM);

   act.sa_handler = terminator;
   act.sa_mask = block;
   act.sa_flags = 0;

   if (sigaction(SIGHUP, &act, NULL) ||
       sigaction(SIGINT, &act, NULL) ||
       sigaction(SIGQUIT, &act, NULL) ||
       sigaction(SIGTERM, &act, NULL)) {
      ERRMSG("est_dpi_terminator", "sigaction", errno);
      exit(1);
   }

   if (atexit(cleanup) != 0) {
      ERRMSG("est_dpi_terminator", "atexit", 0);
      MSG_ERR("Hey! atexit failed, how did that happen?\n");
      exit(1);
   }
}

/*
 * Decode gzipped data
 */
static Dstr *Decode_gzip(Decode *dc, const char *instr, int inlen)
{
   int rc = Z_OK;

   z_stream *zs = (z_stream *)dc->state;

   int inputConsumed = 0;
   Dstr *output = dStr_new("");

   while ((rc == Z_OK) && (inputConsumed < inlen)) {
      zs->next_in = (Bytef *)instr + inputConsumed;
      zs->avail_in = inlen - inputConsumed;

      zs->next_out = (Bytef *)dc->buffer;
      zs->avail_out = bufsize;

      rc = inflate(zs, Z_SYNC_FLUSH);

      dStr_append_l(output, dc->buffer, zs->total_out);

      if ((rc == Z_OK) || (rc == Z_STREAM_END)) {
         // Z_STREAM_END at end of file

         inputConsumed += zs->total_in;
         zs->total_out = 0;
         zs->total_in = 0;
      } else if (rc == Z_DATA_ERROR) {
         MSG_ERR("gzip decompression error\n");
      }
   }
   return output;
}

ump_memory_backend* ump_memory_backend_create ( void )
{
	ump_memory_backend * backend = NULL;

	/* Create the dynamic memory allocator backend */
	if (0 == ump_backend)
	{
		DBG_MSG(2, ("Using dedicated memory backend\n"));

		DBG_MSG(2, ("Requesting dedicated memory: 0x%08x, size: %u\n", ump_memory_address, ump_memory_size));
		/* Ask the OS if we can use the specified physical memory */
		if (NULL == request_mem_region(ump_memory_address, ump_memory_size, "UMP Memory"))
		{
			MSG_ERR(("Failed to request memory region (0x%08X - 0x%08X). Is Mali DD already loaded?\n", ump_memory_address, ump_memory_address + ump_memory_size - 1));
			return NULL;
		}
		backend = ump_block_allocator_create(ump_memory_address, ump_memory_size);
	}
	else if (1 == ump_backend)
	{
		DBG_MSG(2, ("Using OS memory backend, allocation limit: %d\n", ump_memory_size));
		backend = ump_os_memory_backend_create(ump_memory_size);
	}
/* MALI_SEC */
#ifdef CONFIG_UMP_VCM_ALLOC
	else if (2 == ump_backend)
	{
		DBG_MSG(2, ("Using VCM memory backend, allocation limit: %d\n", ump_memory_size));
		backend = ump_vcm_memory_backend_create(ump_memory_size);
	}
#endif

	return backend;
}

/*!                                                                                                
**                                                                                                 
**                                                                                                 
** @param d                                                                                        
** @param cob_id                                                                                   
**                                                                                                 
** @return                                                                                         
**/  
UNS8 sendSlaveLSSMessage(CO_Data* d, UNS8 command,void *dat1,void *dat2)
{
	Message m;
	UNS8 i;

	if (!d->CurrentCommunicationState.csLSS)
	{
		MSG_WAR(0x2D17, "unable to send the LSS message, not in the proper state =>", d->nodeState);
		return 0xFF;
	}

	for (i = 1; i < 8; i++)
	{
		m.data[i] = 0;
	}

	m.len = 8;
	m.rtr = NOT_A_REQUEST;
	m.data[0]=command;
	m.cob_id=UNS16_LE(SLSS_ADRESS);

	/* Tha data sent with the msg depends on the command */
	switch(command)
	{
		case LSS_INQ_NODE_ID: /* Inquire Node-ID */
		{
			m.data[1]=*(UNS8 *)dat1;
		} break;
		case LSS_CONF_NODE_ID: /* Configure Node-ID */
		case LSS_CONF_BIT_TIMING: /* Configure Bit Timing Parameters */
		case LSS_CONF_STORE: /* Store Configured Parameters */
		{
			m.data[1]=*(UNS8 *)dat1;
			m.data[2]=*(UNS8 *)dat2;
		} break;
		case LSS_INQ_VENDOR_ID: /* Inquire Identity Vendor-ID */
		case LSS_INQ_PRODUCT_CODE: /* Inquire Identity Product-Code */
		case LSS_INQ_REV_NUMBER: /* Inquire Identity Revision-Number */
		case LSS_INQ_SERIAL_NUMBER: /* Inquire Identity Serial-Number */
		{
			m.data[1]=(UNS8)(*(UNS32*)dat1 & 0xFF);
			m.data[2]=(UNS8)(*(UNS32*)dat1>>8 & 0xFF);
			m.data[3]=(UNS8)(*(UNS32*)dat1>>16 & 0xFF);
			m.data[4]=(UNS8)(*(UNS32*)dat1>>24 & 0xFF);
		} break;
		case LSS_SM_SELECTIVE_RESP: /* Switch Mode Selective response*/
		case LSS_IDENT_SLAVE: /* LSS Identify Slave */
		case LSS_IDENT_NON_CONF_SLAVE: /* LSS identify non-configured remote slave */
		{
		} break;
		default:
		{
			MSG_ERR(0x1D18, "send Slave LSS command not implemented", command);
			return 0xFF;
		}
	}

	return canSend(d->canHandle,&m);
}

/*
 * Close this socket for reading and writing.
 * (flush pending data)
 */
void a_Dpip_dsh_close(Dsh *dsh)
{
   int st;

   /* flush internal buffer */
   a_Dpip_dsh_write(dsh, 1, "", 0);

   /* close fds */
   st = dClose(dsh->fd_in);
   if (st < 0)
      MSG_ERR("[a_Dpip_dsh_close] close: %s\n", dStrerror(errno));
   if (dsh->fd_out != dsh->fd_in) {
      st = dClose(dsh->fd_out);
      if (st < 0)
         MSG_ERR("[a_Dpip_dsh_close] close: %s\n", dStrerror(errno));
   }
}

/*
 * Close this socket for reading and writing.
 * (flush pending data)
 */
void a_Dpip_dsh_close(Dsh *dsh)
{
   int st;

   /* flush internal buffer */
   a_Dpip_dsh_write(dsh, 1, "", 0);

   /* close fds */
   while((st = close(dsh->fd_in)) < 0 && errno == EINTR) ;
   if (st < 0)
      MSG_ERR("[a_Dpip_dsh_close] close: %s\n", dStrerror(errno));
   if (dsh->fd_out != dsh->fd_in) {
      while((st = close(dsh->fd_out)) < 0 && errno == EINTR) ;
      if (st < 0)
         MSG_ERR("[a_Dpip_dsh_close] close: %s\n", dStrerror(errno));
   }
}