upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-02-27 03:40:37 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions
opnsense-src/sys/dev/qat/qat_api/common/compression/dc_datapath.c
Krzysztof Zdziarski 266b0663c5 qat: Add Intel® 4xxx Series VF driver support 
Overview:
Intel(R) QuickAssist Technology (Intel(R) QAT) provides hardware
acceleration for offloading security, authentication and compression
services from the CPU, thus significantly increasing the performance and
efficiency of standard platform solutions.

This commit introduces:
- Intel® 4xxx Series VF driver support.
- Device configurability via sysctls.
- UIO support for Intel® 4xxx Series devices.

Patch co-authored by: Krzysztof Zdziarski <krzysztofx.zdziarski@intel.com>
Patch co-authored by: Michal Gulbicki <michalx.gulbicki@intel.com>
Patch co-authored by: Julian Grajkowski <julianx.grajkowski@intel.com>
Patch co-authored by: Piotr Kasierski <piotrx.kasierski@intel.com>
Patch co-authored by: Lukasz Kolodzinski <lukaszx.kolodzinski@intel.com>
Patch co-authored by: Karol Grzadziel <karolx.grzadziel@intel.com>

Sponsored by:	Intel Corporation
Differential Revision: https://reviews.freebsd.org/D39850
2023-06-12 13:44:01 -04:00

1986 lines
59 KiB
C
Raw Blame History

/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* $FreeBSD$ */
/**
*****************************************************************************
* @file dc_datapath.c
*
* @defgroup Dc_DataCompression DC Data Compression
*
* @ingroup Dc_DataCompression
*
* @description
* Implementation of the Data Compression datapath operations.
*
*****************************************************************************/
/*
*******************************************************************************
* Include public/global header files
*******************************************************************************
*/
#include "cpa.h"
#include "cpa_dc.h"
#include "cpa_dc_dp.h"
/*
*******************************************************************************
* Include private header files
*******************************************************************************
*/
#include "dc_session.h"
#include "dc_datapath.h"
#include "sal_statistics.h"
#include "lac_common.h"
#include "lac_mem.h"
#include "lac_mem_pools.h"
#include "sal_types_compression.h"
#include "dc_stats.h"
#include "lac_buffer_desc.h"
#include "lac_sal.h"
#include "lac_log.h"
#include "lac_sync.h"
#include "sal_service_state.h"
#include "sal_qat_cmn_msg.h"
#include "sal_hw_gen.h"
#include "dc_error_counter.h"
#define DC_COMP_MAX_BUFF_SIZE (1024 * 64)
static QatUtilsAtomic dcErrorCount[MAX_DC_ERROR_TYPE];
void
dcErrorLog(CpaDcReqStatus dcError)
{
Cpa32U absError = 0;
absError = abs(dcError);
if ((dcError < CPA_DC_OK) && (absError < MAX_DC_ERROR_TYPE)) {
qatUtilsAtomicInc(&(dcErrorCount[absError]));
}
}
Cpa64U
getDcErrorCounter(CpaDcReqStatus dcError)
{
Cpa32U absError = 0;
absError = abs(dcError);
if (!(dcError >= CPA_DC_OK || dcError < CPA_DC_EMPTY_DYM_BLK)) {
return (Cpa64U)qatUtilsAtomicGet(&dcErrorCount[absError]);
}
return 0;
}
static inline void
dcUpdateXltOverflowChecksumsGen4(const dc_compression_cookie_t *pCookie,
const icp_qat_fw_resp_comp_pars_t *pRespPars,
CpaDcRqResults *pDcResults)
{
dc_session_desc_t *pSessionDesc =
DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle);
/* Recompute CRC checksum when either the checksum type
* is CPA_DC_CRC32 or when the integrity CRCs are enabled.
*/
if (CPA_DC_CRC32 == pSessionDesc->checksumType) {
pDcResults->checksum = pRespPars->crc.legacy.curr_crc32;
/* No need to recalculate the swCrc64I here as this will get
* handled later in dcHandleIntegrityChecksumsGen4.
*/
} else if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
pDcResults->checksum = pRespPars->crc.legacy.curr_adler_32;
}
}
void
dcCompression_ProcessCallback(void *pRespMsg)
{
CpaStatus status = CPA_STATUS_SUCCESS;
icp_qat_fw_comp_resp_t *pCompRespMsg = NULL;
void *callbackTag = NULL;
Cpa64U *pReqData = NULL;
CpaDcDpOpData *pResponse = NULL;
CpaDcRqResults *pResults = NULL;
CpaDcCallbackFn pCbFunc = NULL;
dc_session_desc_t *pSessionDesc = NULL;
sal_compression_service_t *pService = NULL;
dc_compression_cookie_t *pCookie = NULL;
CpaDcOpData *pOpData = NULL;
CpaBoolean cmpPass = CPA_TRUE, xlatPass = CPA_TRUE;
CpaBoolean isDcDp = CPA_FALSE;
CpaBoolean integrityCrcCheck = CPA_FALSE;
CpaBoolean verifyHwIntegrityCrcs = CPA_FALSE;
Cpa8U cmpErr = ERR_CODE_NO_ERROR, xlatErr = ERR_CODE_NO_ERROR;
dc_request_dir_t compDecomp = DC_COMPRESSION_REQUEST;
Cpa8U opStatus = ICP_QAT_FW_COMN_STATUS_FLAG_OK;
Cpa8U hdrFlags = 0;
/* Cast response message to compression response message type */
pCompRespMsg = (icp_qat_fw_comp_resp_t *)pRespMsg;
/* Extract request data pointer from the opaque data */
LAC_MEM_SHARED_READ_TO_PTR(pCompRespMsg->opaque_data, pReqData);
/* Extract fields from the request data structure */
pCookie = (dc_compression_cookie_t *)pReqData;
if (!pCookie)
return;
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle);
pService = (sal_compression_service_t *)(pCookie->dcInstance);
isDcDp = pSessionDesc->isDcDp;
if (CPA_TRUE == isDcDp) {
pResponse = (CpaDcDpOpData *)pReqData;
pResults = &(pResponse->results);
if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) {
compDecomp = DC_DECOMPRESSION_REQUEST;
}
pCookie = NULL;
} else {
pResults = pCookie->pResults;
callbackTag = pCookie->callbackTag;
pCbFunc = pCookie->pSessionDesc->pCompressionCb;
compDecomp = pCookie->compDecomp;
pOpData = pCookie->pDcOpData;
}
opStatus = pCompRespMsg->comn_resp.comn_status;
if (NULL != pOpData) {
verifyHwIntegrityCrcs = pOpData->verifyHwIntegrityCrcs;
integrityCrcCheck = pOpData->integrityCrcCheck;
}
hdrFlags = pCompRespMsg->comn_resp.hdr_flags;
/* Get the cmp error code */
cmpErr = pCompRespMsg->comn_resp.comn_error.s1.cmp_err_code;
if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(opStatus)) {
/* Compression not supported by firmware, set produced/consumed
to zero
and call the cb function with status CPA_STATUS_UNSUPPORTED
*/
QAT_UTILS_LOG("Compression feature not supported\n");
status = CPA_STATUS_UNSUPPORTED;
pResults->status = (Cpa8S)cmpErr;
pResults->consumed = 0;
pResults->produced = 0;
if (CPA_TRUE == isDcDp) {
if (pResponse)
pResponse->responseStatus =
CPA_STATUS_UNSUPPORTED;
(pService->pDcDpCb)(pResponse);
} else {
/* Free the memory pool */
Lac_MemPoolEntryFree(pCookie);
pCookie = NULL;
if (NULL != pCbFunc) {
pCbFunc(callbackTag, status);
}
}
if (DC_COMPRESSION_REQUEST == compDecomp) {
COMPRESSION_STAT_INC(numCompCompletedErrors, pService);
} else {
COMPRESSION_STAT_INC(numDecompCompletedErrors,
pService);
}
return;
} else {
/* Check compression response status */
cmpPass =
(CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK ==
ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(opStatus));
}
if (isDcGen2x(pService)) {
/* QAT1.7 and QAT 1.8 hardware */
if (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr) {
cmpPass = CPA_TRUE;
cmpErr = ERR_CODE_NO_ERROR;
}
} else {
/* QAT2.0 hardware cancels the incomplete file errors
* only for DEFLATE algorithm.
* Decompression direction is not tested in the callback as
* the request does not allow it.
*/
if ((pSessionDesc->compType == CPA_DC_DEFLATE) &&
(CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr)) {
cmpPass = CPA_TRUE;
cmpErr = ERR_CODE_NO_ERROR;
}
}
/* log the slice hang and endpoint push/pull error inside the response
*/
if (ERR_CODE_SSM_ERROR == (Cpa8S)cmpErr) {
QAT_UTILS_LOG(
"Slice hang detected on the compression slice.\n");
} else if (ERR_CODE_ENDPOINT_ERROR == (Cpa8S)cmpErr) {
QAT_UTILS_LOG(
"PCIe End Point Push/Pull or TI/RI Parity error detected.\n");
}
/* We return the compression error code for now. We would need to update
* the API if we decide to return both error codes */
pResults->status = (Cpa8S)cmpErr;
/* Check the translator status */
if ((DC_COMPRESSION_REQUEST == compDecomp) &&
(CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) {
/* Check translator response status */
xlatPass =
(CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK ==
ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(opStatus));
/* Get the translator error code */
xlatErr = pCompRespMsg->comn_resp.comn_error.s1.xlat_err_code;
/* Return a fatal error or a potential error in the translator
* slice if the compression slice did not return any error */
if ((CPA_DC_OK == pResults->status) ||
(CPA_DC_FATALERR == (Cpa8S)xlatErr)) {
pResults->status = (Cpa8S)xlatErr;
}
}
/* Update dc error counter */
dcErrorLog(pResults->status);
if (CPA_FALSE == isDcDp) {
/* In case of any error for an end of packet request, we need to
* update
* the request type for the following request */
if (CPA_DC_FLUSH_FINAL == pCookie->flushFlag && cmpPass &&
xlatPass) {
pSessionDesc->requestType = DC_REQUEST_FIRST;
} else {
pSessionDesc->requestType = DC_REQUEST_SUBSEQUENT;
}
if ((CPA_DC_STATEFUL == pSessionDesc->sessState) ||
((CPA_DC_STATELESS == pSessionDesc->sessState) &&
(DC_COMPRESSION_REQUEST == compDecomp))) {
/* Overflow is a valid use case for Traditional API
* only. Stateful Overflow is supported in both
* compression and decompression direction. Stateless
* Overflow is supported only in compression direction.
*/
if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr)
cmpPass = CPA_TRUE;
if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) {
if (isDcGen4x(pService) &&
(CPA_TRUE ==
pService->comp_device_data
.translatorOverflow)) {
pResults->consumed =
pCompRespMsg->comp_resp_pars
.input_byte_counter;
dcUpdateXltOverflowChecksumsGen4(
pCookie,
&pCompRespMsg->comp_resp_pars,
pResults);
}
xlatPass = CPA_TRUE;
}
}
} else {
if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr) {
cmpPass = CPA_FALSE;
}
if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) {
/* XLT overflow is not valid for Data Plane requests */
xlatPass = CPA_FALSE;
}
}
if ((CPA_TRUE == cmpPass) && (CPA_TRUE == xlatPass)) {
/* Extract the response from the firmware */
pResults->consumed =
pCompRespMsg->comp_resp_pars.input_byte_counter;
pResults->produced =
pCompRespMsg->comp_resp_pars.output_byte_counter;
pSessionDesc->cumulativeConsumedBytes += pResults->consumed;
/* Handle Checksum for end to end data integrity. */
if (CPA_TRUE ==
pService->generic_service_info.integrityCrcCheck &&
CPA_TRUE == integrityCrcCheck) {
pSessionDesc->previousChecksum =
pSessionDesc->seedSwCrc.swCrc32I;
} else if (CPA_DC_OVERFLOW != (Cpa8S)xlatErr) {
if (CPA_DC_CRC32 == pSessionDesc->checksumType) {
pResults->checksum =
pCompRespMsg->comp_resp_pars.crc.legacy
.curr_crc32;
} else if (CPA_DC_ADLER32 ==
pSessionDesc->checksumType) {
pResults->checksum =
pCompRespMsg->comp_resp_pars.crc.legacy
.curr_adler_32;
}
pSessionDesc->previousChecksum = pResults->checksum;
}
if (DC_DECOMPRESSION_REQUEST == compDecomp) {
pResults->endOfLastBlock =
(ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET ==
ICP_QAT_FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(
opStatus));
} else {
/* Check if returned data is a stored block
* in compression direction
*/
pResults->dataUncompressed =
ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_GET(hdrFlags);
}
/* Save the checksum for the next request */
if ((CPA_DC_OVERFLOW != (Cpa8S)xlatErr) &&
(CPA_TRUE == verifyHwIntegrityCrcs)) {
pSessionDesc->previousChecksum =
pSessionDesc->seedSwCrc.swCrc32I;
}
/* Check if a CNV recovery happened and
* increase stats counter
*/
if ((DC_COMPRESSION_REQUEST == compDecomp) &&
ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(hdrFlags) &&
ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_GET(hdrFlags)) {
COMPRESSION_STAT_INC(numCompCnvErrorsRecovered,
pService);
}
if (CPA_TRUE == isDcDp) {
if (pResponse)
pResponse->responseStatus = CPA_STATUS_SUCCESS;
} else {
if (DC_COMPRESSION_REQUEST == compDecomp) {
COMPRESSION_STAT_INC(numCompCompleted,
pService);
} else {
COMPRESSION_STAT_INC(numDecompCompleted,
pService);
}
}
} else {
#ifdef ICP_DC_RETURN_COUNTERS_ON_ERROR
/* Extract the response from the firmware */
pResults->consumed =
pCompRespMsg->comp_resp_pars.input_byte_counter;
pResults->produced =
pCompRespMsg->comp_resp_pars.output_byte_counter;
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
pSessionDesc->cumulativeConsumedBytes +=
pResults->consumed;
} else {
/* In the stateless case all requests have both SOP and
* EOP set */
pSessionDesc->cumulativeConsumedBytes =
pResults->consumed;
}
#else
pResults->consumed = 0;
pResults->produced = 0;
#endif
if (CPA_DC_OVERFLOW == pResults->status &&
CPA_DC_STATELESS == pSessionDesc->sessState) {
/* This error message will be returned by Data Plane API
* in both
* compression and decompression direction. With
* Traditional API
* this error message will be returned only in stateless
* decompression direction */
QAT_UTILS_LOG(
"Unrecoverable error: stateless overflow. You may need to increase the size of your destination buffer.\n");
}
if (CPA_TRUE == isDcDp) {
if (pResponse)
pResponse->responseStatus = CPA_STATUS_FAIL;
} else {
if (CPA_DC_OK != pResults->status &&
CPA_DC_INCOMPLETE_FILE_ERR != pResults->status) {
status = CPA_STATUS_FAIL;
}
if (DC_COMPRESSION_REQUEST == compDecomp) {
COMPRESSION_STAT_INC(numCompCompletedErrors,
pService);
} else {
COMPRESSION_STAT_INC(numDecompCompletedErrors,
pService);
}
}
}
if (CPA_TRUE == isDcDp) {
/* Decrement number of stateless pending callbacks for session
*/
pSessionDesc->pendingDpStatelessCbCount--;
(pService->pDcDpCb)(pResponse);
} else {
/* Decrement number of pending callbacks for session */
if (CPA_DC_STATELESS == pSessionDesc->sessState) {
qatUtilsAtomicDec(
&(pCookie->pSessionDesc->pendingStatelessCbCount));
} else if (0 !=
qatUtilsAtomicGet(&pCookie->pSessionDesc
->pendingStatefulCbCount)) {
qatUtilsAtomicDec(
&(pCookie->pSessionDesc->pendingStatefulCbCount));
}
/* Free the memory pool */
Lac_MemPoolEntryFree(pCookie);
pCookie = NULL;
if (NULL != pCbFunc) {
pCbFunc(callbackTag, status);
}
}
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Check that all the parameters in the pOpData structure are valid
*
* @description
* Check that all the parameters in the pOpData structure are valid
*
* @param[in] pService Pointer to the compression service
* @param[in] pOpData Pointer to request information structure
* holding parameters for cpaDcCompress2 and
* CpaDcDecompressData2
* @retval CPA_STATUS_SUCCESS Function executed successfully
* @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
*
*****************************************************************************/
CpaStatus
dcCheckOpData(sal_compression_service_t *pService, CpaDcOpData *pOpData)
{
CpaDcSkipMode skipMode = 0;
if ((pOpData->flushFlag < CPA_DC_FLUSH_NONE) ||
(pOpData->flushFlag > CPA_DC_FLUSH_FULL)) {
LAC_INVALID_PARAM_LOG("Invalid flushFlag value");
return CPA_STATUS_INVALID_PARAM;
}
skipMode = pOpData->inputSkipData.skipMode;
if ((skipMode < CPA_DC_SKIP_DISABLED) ||
(skipMode > CPA_DC_SKIP_STRIDE)) {
LAC_INVALID_PARAM_LOG("Invalid input skip mode value");
return CPA_STATUS_INVALID_PARAM;
}
skipMode = pOpData->outputSkipData.skipMode;
if ((skipMode < CPA_DC_SKIP_DISABLED) ||
(skipMode > CPA_DC_SKIP_STRIDE)) {
LAC_INVALID_PARAM_LOG("Invalid output skip mode value");
return CPA_STATUS_INVALID_PARAM;
}
if (pOpData->integrityCrcCheck == CPA_FALSE &&
pOpData->verifyHwIntegrityCrcs == CPA_TRUE) {
LAC_INVALID_PARAM_LOG(
"integrityCrcCheck must be set to true"
"in order to enable verifyHwIntegrityCrcs");
return CPA_STATUS_INVALID_PARAM;
}
if (pOpData->integrityCrcCheck != CPA_TRUE &&
pOpData->integrityCrcCheck != CPA_FALSE) {
LAC_INVALID_PARAM_LOG("Invalid integrityCrcCheck value");
return CPA_STATUS_INVALID_PARAM;
}
if (pOpData->verifyHwIntegrityCrcs != CPA_TRUE &&
pOpData->verifyHwIntegrityCrcs != CPA_FALSE) {
LAC_INVALID_PARAM_LOG("Invalid verifyHwIntegrityCrcs value");
return CPA_STATUS_INVALID_PARAM;
}
if (pOpData->compressAndVerify != CPA_TRUE &&
pOpData->compressAndVerify != CPA_FALSE) {
LAC_INVALID_PARAM_LOG("Invalid cnv decompress check value");
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_TRUE == pOpData->integrityCrcCheck &&
CPA_FALSE == pService->generic_service_info.integrityCrcCheck) {
LAC_INVALID_PARAM_LOG("Integrity CRC check is not "
"supported on this device");
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_TRUE == pOpData->integrityCrcCheck &&
NULL == pOpData->pCrcData) {
LAC_INVALID_PARAM_LOG("Integrity CRC data structure "
"not intialized in CpaDcOpData");
return CPA_STATUS_INVALID_PARAM;
}
return CPA_STATUS_SUCCESS;
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Check the compression source buffer for Batch and Pack API.
*
* @description
* Check that all the parameters used for Pack compression
* request are valid. This function essentially checks the source buffer
* parameters and results structure parameters.
*
* @param[in] pSessionHandle Session handle
* @param[in] pSrcBuff Pointer to data buffer for compression
* @param[in] pDestBuff Pointer to buffer space allocated for
* output data
* @param[in] pResults Pointer to results structure
* @param[in] flushFlag Indicates the type of flush to be
* performed
* @param[in] srcBuffSize Size of the source buffer
*
* @retval CPA_STATUS_SUCCESS Function executed successfully
* @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
*
*****************************************************************************/
static CpaStatus
dcCheckSourceData(CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcRqResults *pResults,
CpaDcFlush flushFlag,
Cpa64U srcBuffSize,
CpaDcSkipData *skipData)
{
dc_session_desc_t *pSessionDesc = NULL;
LAC_CHECK_NULL_PARAM(pSessionHandle);
LAC_CHECK_NULL_PARAM(pSrcBuff);
LAC_CHECK_NULL_PARAM(pDestBuff);
LAC_CHECK_NULL_PARAM(pResults);
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
if (NULL == pSessionDesc) {
LAC_INVALID_PARAM_LOG("Session handle not as expected");
return CPA_STATUS_INVALID_PARAM;
}
if ((flushFlag < CPA_DC_FLUSH_NONE) ||
(flushFlag > CPA_DC_FLUSH_FULL)) {
LAC_INVALID_PARAM_LOG("Invalid flushFlag value");
return CPA_STATUS_INVALID_PARAM;
}
if (pSrcBuff == pDestBuff) {
LAC_INVALID_PARAM_LOG("In place operation not supported");
return CPA_STATUS_INVALID_PARAM;
}
/* Compressing zero bytes is not supported for stateless sessions
* for non Batch and Pack requests */
if ((CPA_DC_STATELESS == pSessionDesc->sessState) &&
(0 == srcBuffSize) && (NULL == skipData)) {
LAC_INVALID_PARAM_LOG(
"The source buffer size needs to be greater than "
"zero bytes for stateless sessions");
return CPA_STATUS_INVALID_PARAM;
}
if (srcBuffSize > DC_BUFFER_MAX_SIZE) {
LAC_INVALID_PARAM_LOG(
"The source buffer size needs to be less than or "
"equal to 2^32-1 bytes");
return CPA_STATUS_INVALID_PARAM;
}
return CPA_STATUS_SUCCESS;
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Check the compression or decompression function parameters.
*
* @description
* Check that all the parameters used for a Batch and Pack compression
* request are valid. This function essentially checks the destination
* buffer parameters and intermediate buffer parameters.
*
* @param[in] pService Pointer to the compression service
* @param[in] pSessionHandle Session handle
* @param[in] pDestBuff Pointer to buffer space allocated for
* output data
* @param[in] compDecomp Direction of the operation
*
* @retval CPA_STATUS_SUCCESS Function executed successfully
* @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
*
*****************************************************************************/
static CpaStatus
dcCheckDestinationData(sal_compression_service_t *pService,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pDestBuff,
dc_request_dir_t compDecomp)
{
dc_session_desc_t *pSessionDesc = NULL;
Cpa64U destBuffSize = 0;
LAC_CHECK_NULL_PARAM(pSessionHandle);
LAC_CHECK_NULL_PARAM(pDestBuff);
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
if (NULL == pSessionDesc) {
LAC_INVALID_PARAM_LOG("Session handle not as expected");
return CPA_STATUS_INVALID_PARAM;
}
if (LacBuffDesc_BufferListVerify(pDestBuff,
&destBuffSize,
LAC_NO_ALIGNMENT_SHIFT) !=
CPA_STATUS_SUCCESS) {
LAC_INVALID_PARAM_LOG(
"Invalid destination buffer list parameter");
return CPA_STATUS_INVALID_PARAM;
}
if (destBuffSize > DC_BUFFER_MAX_SIZE) {
LAC_INVALID_PARAM_LOG(
"The destination buffer size needs to be less "
"than or equal to 2^32-1 bytes");
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_TRUE == pSessionDesc->isDcDp) {
LAC_INVALID_PARAM_LOG(
"The session type should not be data plane");
return CPA_STATUS_INVALID_PARAM;
}
if (DC_COMPRESSION_REQUEST == compDecomp) {
if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) {
/* Check if intermediate buffers are supported */
if ((isDcGen2x(pService)) &&
((0 == pService->pInterBuffPtrsArrayPhyAddr) ||
(NULL == pService->pInterBuffPtrsArray))) {
LAC_LOG_ERROR(
"No intermediate buffer defined for this instance "
"- see cpaDcStartInstance");
return CPA_STATUS_INVALID_PARAM;
}
/* Ensure that the destination buffer size is greater or
* equal to 128B */
if (destBuffSize < DC_DEST_BUFFER_DYN_MIN_SIZE) {
LAC_INVALID_PARAM_LOG(
"Destination buffer size should be "
"greater or equal to 128B");
return CPA_STATUS_INVALID_PARAM;
}
} else
{
/* Ensure that the destination buffer size is greater or
* equal to devices min output buff size */
if (destBuffSize <
pService->comp_device_data.minOutputBuffSize) {
LAC_INVALID_PARAM_LOG1(
"Destination buffer size should be "
"greater or equal to %d bytes",
pService->comp_device_data
.minOutputBuffSize);
return CPA_STATUS_INVALID_PARAM;
}
}
} else {
/* Ensure that the destination buffer size is greater than
* 0 bytes */
if (destBuffSize < DC_DEST_BUFFER_DEC_MIN_SIZE) {
LAC_INVALID_PARAM_LOG(
"Destination buffer size should be "
"greater than 0 bytes");
return CPA_STATUS_INVALID_PARAM;
}
}
return CPA_STATUS_SUCCESS;
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Populate the compression request parameters
*
* @description
* This function will populate the compression request parameters
*
* @param[out] pCompReqParams Pointer to the compression request parameters
* @param[in] pCookie Pointer to the compression cookie
*
*****************************************************************************/
static void
dcCompRequestParamsPopulate(icp_qat_fw_comp_req_params_t *pCompReqParams,
dc_compression_cookie_t *pCookie)
{
pCompReqParams->comp_len = pCookie->srcTotalDataLenInBytes;
pCompReqParams->out_buffer_sz = pCookie->dstTotalDataLenInBytes;
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Create the requests for compression or decompression
*
* @description
* Create the requests for compression or decompression. This function
* will update the cookie will all required information.
*
* @param{out] pCookie Pointer to the compression cookie
* @param[in] pService Pointer to the compression service
* @param[in] pSessionDesc Pointer to the session descriptor
* @param[in pSessionHandle Session handle
* @param[in] pSrcBuff Pointer to data buffer for compression
* @param[in] pDestBuff Pointer to buffer space for data after
* compression
* @param[in] pResults Pointer to results structure
* @param[in] flushFlag Indicates the type of flush to be
* performed
* @param[in] pOpData Pointer to request information structure
* holding parameters for cpaDcCompress2
* and CpaDcDecompressData2
* @param[in] callbackTag Pointer to the callback tag
* @param[in] compDecomp Direction of the operation
* @param[in] compressAndVerify Compress and Verify
*
* @retval CPA_STATUS_SUCCESS Function executed successfully
* @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
*
*****************************************************************************/
static CpaStatus
dcCreateRequest(dc_compression_cookie_t *pCookie,
sal_compression_service_t *pService,
dc_session_desc_t *pSessionDesc,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcRqResults *pResults,
CpaDcFlush flushFlag,
CpaDcOpData *pOpData,
void *callbackTag,
dc_request_dir_t compDecomp,
dc_cnv_mode_t cnvMode)
{
icp_qat_fw_comp_req_t *pMsg = NULL;
icp_qat_fw_comp_req_params_t *pCompReqParams = NULL;
Cpa64U srcAddrPhys = 0, dstAddrPhys = 0;
Cpa64U srcTotalDataLenInBytes = 0, dstTotalDataLenInBytes = 0;
Cpa32U rpCmdFlags = 0;
Cpa8U sop = ICP_QAT_FW_COMP_SOP;
Cpa8U eop = ICP_QAT_FW_COMP_EOP;
Cpa8U bFinal = ICP_QAT_FW_COMP_NOT_BFINAL;
Cpa8U crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY;
Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV;
Cpa8U cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY;
CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX;
CpaBoolean integrityCrcCheck = CPA_FALSE;
CpaStatus status = CPA_STATUS_SUCCESS;
CpaDcFlush flush = CPA_DC_FLUSH_NONE;
Cpa32U initial_adler = 1;
Cpa32U initial_crc32 = 0;
icp_qat_fw_comp_req_t *pReqCache = NULL;
/* Write the buffer descriptors */
status = LacBuffDesc_BufferListDescWriteAndGetSize(
pSrcBuff,
&srcAddrPhys,
CPA_FALSE,
&srcTotalDataLenInBytes,
&(pService->generic_service_info));
if (status != CPA_STATUS_SUCCESS) {
return status;
}
status = LacBuffDesc_BufferListDescWriteAndGetSize(
pDestBuff,
&dstAddrPhys,
CPA_FALSE,
&dstTotalDataLenInBytes,
&(pService->generic_service_info));
if (status != CPA_STATUS_SUCCESS) {
return status;
}
/* Populate the compression cookie */
pCookie->dcInstance = pService;
pCookie->pSessionHandle = pSessionHandle;
pCookie->callbackTag = callbackTag;
pCookie->pSessionDesc = pSessionDesc;
pCookie->pDcOpData = pOpData;
pCookie->pResults = pResults;
pCookie->compDecomp = compDecomp;
pCookie->pUserSrcBuff = NULL;
pCookie->pUserDestBuff = NULL;
/* Extract flush flag from either the opData or from the
* parameter. Opdata have been introduce with APIs
* cpaDcCompressData2 and cpaDcDecompressData2 */
if (NULL != pOpData) {
flush = pOpData->flushFlag;
integrityCrcCheck = pOpData->integrityCrcCheck;
} else {
flush = flushFlag;
}
pCookie->flushFlag = flush;
/* The firmware expects the length in bytes for source and destination
* to be Cpa32U parameters. However the total data length could be
* bigger as allocated by the user. We ensure that this is not the case
* in dcCheckSourceData and cast the values to Cpa32U here */
pCookie->srcTotalDataLenInBytes = (Cpa32U)srcTotalDataLenInBytes;
if ((isDcGen2x(pService)) && (DC_COMPRESSION_REQUEST == compDecomp) &&
(CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) {
if (pService->minInterBuffSizeInBytes <
(Cpa32U)dstTotalDataLenInBytes) {
pCookie->dstTotalDataLenInBytes =
(Cpa32U)(pService->minInterBuffSizeInBytes);
} else {
pCookie->dstTotalDataLenInBytes =
(Cpa32U)dstTotalDataLenInBytes;
}
} else
{
pCookie->dstTotalDataLenInBytes =
(Cpa32U)dstTotalDataLenInBytes;
}
/* Device can not decompress an odd byte decompression request
* if bFinal is not set
*/
if (CPA_TRUE != pService->comp_device_data.oddByteDecompNobFinal) {
if ((CPA_DC_STATEFUL == pSessionDesc->sessState) &&
(CPA_DC_FLUSH_FINAL != flushFlag) &&
(DC_DECOMPRESSION_REQUEST == compDecomp) &&
(pCookie->srcTotalDataLenInBytes & 0x1)) {
pCookie->srcTotalDataLenInBytes--;
}
}
/* Device can not decompress odd byte interim requests */
if (CPA_TRUE != pService->comp_device_data.oddByteDecompInterim) {
if ((CPA_DC_STATEFUL == pSessionDesc->sessState) &&
(CPA_DC_FLUSH_FINAL != flushFlag) &&
(CPA_DC_FLUSH_FULL != flushFlag) &&
(DC_DECOMPRESSION_REQUEST == compDecomp) &&
(pCookie->srcTotalDataLenInBytes & 0x1)) {
pCookie->srcTotalDataLenInBytes--;
}
}
pMsg = (icp_qat_fw_comp_req_t *)&pCookie->request;
if (DC_COMPRESSION_REQUEST == compDecomp) {
pReqCache = &(pSessionDesc->reqCacheComp);
} else {
pReqCache = &(pSessionDesc->reqCacheDecomp);
}
/* Fills the msg from the template cached in the session descriptor */
memcpy((void *)pMsg,
(void *)(pReqCache),
LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES);
if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
initial_adler = 1;
initial_crc32 = 0;
if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
pSessionDesc->previousChecksum = initial_adler;
} else {
pSessionDesc->previousChecksum = initial_crc32;
}
} else if (CPA_DC_STATELESS == pSessionDesc->sessState) {
pSessionDesc->previousChecksum = pResults->checksum;
if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
initial_adler = pSessionDesc->previousChecksum;
} else {
initial_crc32 = pSessionDesc->previousChecksum;
}
}
/* Backup source and destination buffer addresses,
* CRC calculations both for CNV and translator overflow
* will be performed on them in the callback function.
*/
pCookie->pUserSrcBuff = pSrcBuff;
pCookie->pUserDestBuff = pDestBuff;
/*
* Due to implementation of CNV support and need for backwards
* compatibility certain fields in the request and response structs had
* been changed, moved or placed in unions cnvMode flag signifies fields
* to be selected from req/res
*
* Doing extended crc checks makes sense only when we want to do the
* actual CNV
*/
if (CPA_TRUE == pService->generic_service_info.integrityCrcCheck &&
CPA_TRUE == integrityCrcCheck) {
pMsg->comp_pars.crc.crc_data_addr =
pSessionDesc->physDataIntegrityCrcs;
crcMode = ICP_QAT_FW_COMP_CRC_MODE_E2E;
} else {
/* Legacy request structure */
pMsg->comp_pars.crc.legacy.initial_adler = initial_adler;
pMsg->comp_pars.crc.legacy.initial_crc32 = initial_crc32;
crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY;
}
/* Populate the cmdFlags */
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
pSessionDesc->previousRequestType = pSessionDesc->requestType;
if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
/* Update the request type for following requests */
pSessionDesc->requestType = DC_REQUEST_SUBSEQUENT;
/* Reinitialise the cumulative amount of consumed bytes
*/
pSessionDesc->cumulativeConsumedBytes = 0;
if (DC_COMPRESSION_REQUEST == compDecomp) {
pSessionDesc->isSopForCompressionProcessed =
CPA_TRUE;
} else if (DC_DECOMPRESSION_REQUEST == compDecomp) {
pSessionDesc->isSopForDecompressionProcessed =
CPA_TRUE;
}
} else {
if (DC_COMPRESSION_REQUEST == compDecomp) {
if (CPA_TRUE ==
pSessionDesc
->isSopForCompressionProcessed) {
sop = ICP_QAT_FW_COMP_NOT_SOP;
} else {
pSessionDesc
->isSopForCompressionProcessed =
CPA_TRUE;
}
} else if (DC_DECOMPRESSION_REQUEST == compDecomp) {
if (CPA_TRUE ==
pSessionDesc
->isSopForDecompressionProcessed) {
sop = ICP_QAT_FW_COMP_NOT_SOP;
} else {
pSessionDesc
->isSopForDecompressionProcessed =
CPA_TRUE;
}
}
}
if ((CPA_DC_FLUSH_FINAL == flush) ||
(CPA_DC_FLUSH_FULL == flush)) {
/* Update the request type for following requests */
pSessionDesc->requestType = DC_REQUEST_FIRST;
} else {
eop = ICP_QAT_FW_COMP_NOT_EOP;
}
} else {
if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
/* Reinitialise the cumulative amount of consumed bytes
*/
pSessionDesc->cumulativeConsumedBytes = 0;
}
}
/* (LW 14 - 15) */
pCompReqParams = &(pMsg->comp_pars);
dcCompRequestParamsPopulate(pCompReqParams, pCookie);
if (CPA_DC_FLUSH_FINAL == flush) {
bFinal = ICP_QAT_FW_COMP_BFINAL;
}
switch (cnvMode) {
case DC_CNVNR:
cnvRecovery = ICP_QAT_FW_COMP_CNV_RECOVERY;
/* Fall through is intended here, because for CNVNR
* cnvDecompReq also needs to be set */
case DC_CNV:
cnvDecompReq = ICP_QAT_FW_COMP_CNV;
if (isDcGen4x(pService)) {
cnvErrorInjection = pSessionDesc->cnvErrorInjection;
}
break;
case DC_NO_CNV:
cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV;
cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY;
break;
}
/* LW 18 */
rpCmdFlags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop,
eop,
bFinal,
cnvDecompReq,
cnvRecovery,
cnvErrorInjection,
crcMode);
pMsg->comp_pars.req_par_flags = rpCmdFlags;
/* Populates the QAT common request middle part of the message
* (LW 6 to 11) */
SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pMsg,
pCookie,
DC_DEFAULT_QAT_PTR_TYPE,
srcAddrPhys,
dstAddrPhys,
0,
0);
return CPA_STATUS_SUCCESS;
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Send a compression request to QAT
*
* @description
* Send the requests for compression or decompression to QAT
*
* @param{in] pCookie Pointer to the compression cookie
* @param[in] pService Pointer to the compression service
* @param[in] pSessionDesc Pointer to the session descriptor
* @param[in] compDecomp Direction of the operation
*
* @retval CPA_STATUS_SUCCESS Function executed successfully
* @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
*
*****************************************************************************/
static CpaStatus
dcSendRequest(dc_compression_cookie_t *pCookie,
sal_compression_service_t *pService,
dc_session_desc_t *pSessionDesc,
dc_request_dir_t compDecomp)
{
CpaStatus status = CPA_STATUS_SUCCESS;
/* Send to QAT */
status = icp_adf_transPutMsg(pService->trans_handle_compression_tx,
(void *)&(pCookie->request),
LAC_QAT_DC_REQ_SZ_LW);
if ((CPA_DC_STATEFUL == pSessionDesc->sessState) &&
(CPA_STATUS_RETRY == status)) {
/* reset requestType after receiving an retry on
* the stateful request */
pSessionDesc->requestType = pSessionDesc->previousRequestType;
}
return status;
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Process the synchronous and asynchronous case for compression or
* decompression
*
* @description
* Process the synchronous and asynchronous case for compression or
* decompression. This function will then create and send the request to
* the firmware.
*
* @param[in] pService Pointer to the compression service
* @param[in] pSessionDesc Pointer to the session descriptor
* @param[in] dcInstance Instance handle derived from discovery
* functions
* @param[in] pSessionHandle Session handle
* @param[in] numRequests Number of operations in the batch request
* @param[in] pBatchOpData Address of the list of jobs to be processed
* @param[in] pSrcBuff Pointer to data buffer for compression
* @param[in] pDestBuff Pointer to buffer space for data after
* compression
* @param[in] pResults Pointer to results structure
* @param[in] flushFlag Indicates the type of flush to be
* performed
* @param[in] pOpData Pointer to request information structure
* holding parameters for cpaDcCompress2 and
* CpaDcDecompressData2
* @param[in] callbackTag Pointer to the callback tag
* @param[in] compDecomp Direction of the operation
* @param[in] isAsyncMode Used to know if synchronous or asynchronous
* mode
* @param[in] cnvMode CNV Mode
*
* @retval CPA_STATUS_SUCCESS Function executed successfully
* @retval CPA_STATUS_RETRY Retry operation
* @retval CPA_STATUS_FAIL Function failed
* @retval CPA_STATUS_RESOURCE Resource error
*
*****************************************************************************/
static CpaStatus
dcCompDecompData(sal_compression_service_t *pService,
dc_session_desc_t *pSessionDesc,
CpaInstanceHandle dcInstance,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcRqResults *pResults,
CpaDcFlush flushFlag,
CpaDcOpData *pOpData,
void *callbackTag,
dc_request_dir_t compDecomp,
CpaBoolean isAsyncMode,
dc_cnv_mode_t cnvMode)
{
CpaStatus status = CPA_STATUS_SUCCESS;
dc_compression_cookie_t *pCookie = NULL;
if ((LacSync_GenWakeupSyncCaller == pSessionDesc->pCompressionCb) &&
isAsyncMode == CPA_TRUE) {
lac_sync_op_data_t *pSyncCallbackData = NULL;
status = LacSync_CreateSyncCookie(&pSyncCallbackData);
if (CPA_STATUS_SUCCESS == status) {
status = dcCompDecompData(pService,
pSessionDesc,
dcInstance,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
pOpData,
pSyncCallbackData,
compDecomp,
CPA_FALSE,
cnvMode);
} else {
return status;
}
if (CPA_STATUS_SUCCESS == status) {
CpaStatus syncStatus = CPA_STATUS_SUCCESS;
syncStatus =
LacSync_WaitForCallback(pSyncCallbackData,
DC_SYNC_CALLBACK_TIMEOUT,
&status,
NULL);
/* If callback doesn't come back */
if (CPA_STATUS_SUCCESS != syncStatus) {
if (DC_COMPRESSION_REQUEST == compDecomp) {
COMPRESSION_STAT_INC(
numCompCompletedErrors, pService);
} else {
COMPRESSION_STAT_INC(
numDecompCompletedErrors, pService);
}
LAC_LOG_ERROR("Callback timed out");
status = syncStatus;
}
} else {
/* As the Request was not sent the Callback will never
* be called, so need to indicate that we're finished
* with cookie so it can be destroyed. */
LacSync_SetSyncCookieComplete(pSyncCallbackData);
}
LacSync_DestroySyncCookie(&pSyncCallbackData);
return status;
}
/* Allocate the compression cookie
* The memory is freed in callback or in sendRequest if an error occurs
*/
pCookie = (dc_compression_cookie_t *)Lac_MemPoolEntryAlloc(
pService->compression_mem_pool);
if (NULL == pCookie) {
LAC_LOG_ERROR("Cannot get mem pool entry for compression");
status = CPA_STATUS_RESOURCE;
} else if ((void *)CPA_STATUS_RETRY == pCookie) {
pCookie = NULL;
status = CPA_STATUS_RETRY;
}
if (CPA_STATUS_SUCCESS == status) {
status = dcCreateRequest(pCookie,
pService,
pSessionDesc,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
pOpData,
callbackTag,
compDecomp,
cnvMode);
}
if (CPA_STATUS_SUCCESS == status) {
/* Increment number of pending callbacks for session */
if (CPA_DC_STATELESS == pSessionDesc->sessState) {
qatUtilsAtomicInc(
&(pSessionDesc->pendingStatelessCbCount));
}
status =
dcSendRequest(pCookie, pService, pSessionDesc, compDecomp);
}
if (CPA_STATUS_SUCCESS == status) {
if (DC_COMPRESSION_REQUEST == compDecomp) {
COMPRESSION_STAT_INC(numCompRequests, pService);
} else {
COMPRESSION_STAT_INC(numDecompRequests, pService);
}
} else {
if (DC_COMPRESSION_REQUEST == compDecomp) {
COMPRESSION_STAT_INC(numCompRequestsErrors, pService);
} else {
COMPRESSION_STAT_INC(numDecompRequestsErrors, pService);
}
/* Decrement number of pending callbacks for session */
if (CPA_DC_STATELESS == pSessionDesc->sessState) {
qatUtilsAtomicDec(
&(pSessionDesc->pendingStatelessCbCount));
} else {
qatUtilsAtomicDec(
&(pSessionDesc->pendingStatefulCbCount));
}
/* Free the memory pool */
if (NULL != pCookie) {
if (status != CPA_STATUS_UNSUPPORTED) {
/* Free the memory pool */
Lac_MemPoolEntryFree(pCookie);
pCookie = NULL;
}
}
}
return status;
}
/**
*****************************************************************************
* @ingroup Dc_DataCompression
* Handle zero length compression or decompression requests
*
* @description
* Handle zero length compression or decompression requests
*
* @param[in] pService Pointer to the compression service
* @param[in] pSessionDesc Pointer to the session descriptor
* @param[in] pResults Pointer to results structure
* @param[in] flushFlag Indicates the type of flush to be
* performed
* @param[in] callbackTag User supplied value to help correlate
* the callback with its associated request
* @param[in] compDecomp Direction of the operation
*
* @retval CPA_TRUE Zero length SOP or MOP processed
* @retval CPA_FALSE Zero length EOP
*
*****************************************************************************/
static CpaStatus
dcZeroLengthRequests(sal_compression_service_t *pService,
dc_session_desc_t *pSessionDesc,
CpaDcRqResults *pResults,
CpaDcFlush flushFlag,
void *callbackTag,
dc_request_dir_t compDecomp)
{
CpaBoolean status = CPA_FALSE;
CpaDcCallbackFn pCbFunc = pSessionDesc->pCompressionCb;
if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
/* Reinitialise the cumulative amount of consumed bytes */
pSessionDesc->cumulativeConsumedBytes = 0;
/* Zero length SOP */
if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
pResults->checksum = 1;
} else {
pResults->checksum = 0;
}
status = CPA_TRUE;
} else if ((CPA_DC_FLUSH_NONE == flushFlag) ||
(CPA_DC_FLUSH_SYNC == flushFlag)) {
/* Zero length MOP */
pResults->checksum = pSessionDesc->previousChecksum;
status = CPA_TRUE;
}
if (CPA_TRUE == status) {
pResults->status = CPA_DC_OK;
pResults->produced = 0;
pResults->consumed = 0;
/* Increment statistics */
if (DC_COMPRESSION_REQUEST == compDecomp) {
COMPRESSION_STAT_INC(numCompRequests, pService);
COMPRESSION_STAT_INC(numCompCompleted, pService);
} else {
COMPRESSION_STAT_INC(numDecompRequests, pService);
COMPRESSION_STAT_INC(numDecompCompleted, pService);
}
LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
if ((NULL != pCbFunc) &&
(LacSync_GenWakeupSyncCaller != pCbFunc)) {
pCbFunc(callbackTag, CPA_STATUS_SUCCESS);
}
return CPA_TRUE;
}
return CPA_FALSE;
}
static CpaStatus
dcParamCheck(CpaInstanceHandle dcInstance,
CpaDcSessionHandle pSessionHandle,
sal_compression_service_t *pService,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcRqResults *pResults,
dc_session_desc_t *pSessionDesc,
CpaDcFlush flushFlag,
Cpa64U srcBuffSize)
{
if (dcCheckSourceData(pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
srcBuffSize,
NULL) != CPA_STATUS_SUCCESS) {
return CPA_STATUS_INVALID_PARAM;
}
if (dcCheckDestinationData(
pService, pSessionHandle, pDestBuff, DC_COMPRESSION_REQUEST) !=
CPA_STATUS_SUCCESS) {
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) {
LAC_INVALID_PARAM_LOG("Invalid sessDirection value");
return CPA_STATUS_INVALID_PARAM;
}
return CPA_STATUS_SUCCESS;
}
CpaStatus
cpaDcCompressData(CpaInstanceHandle dcInstance,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcRqResults *pResults,
CpaDcFlush flushFlag,
void *callbackTag)
{
sal_compression_service_t *pService = NULL;
dc_session_desc_t *pSessionDesc = NULL;
CpaInstanceHandle insHandle = NULL;
Cpa64U srcBuffSize = 0;
if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
insHandle = dcGetFirstHandle();
} else {
insHandle = dcInstance;
}
pService = (sal_compression_service_t *)insHandle;
LAC_CHECK_NULL_PARAM(insHandle);
LAC_CHECK_NULL_PARAM(pSessionHandle);
/* Check if SAL is initialised otherwise return an error */
SAL_RUNNING_CHECK(insHandle);
/* This check is outside the parameter checking as it is needed to
* manage zero length requests */
if (LacBuffDesc_BufferListVerifyNull(pSrcBuff,
&srcBuffSize,
LAC_NO_ALIGNMENT_SHIFT) !=
CPA_STATUS_SUCCESS) {
LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter");
return CPA_STATUS_INVALID_PARAM;
}
/* Ensure this is a compression instance */
SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
if (CPA_STATUS_SUCCESS !=
dcParamCheck(insHandle,
pSessionHandle,
pService,
pSrcBuff,
pDestBuff,
pResults,
pSessionDesc,
flushFlag,
srcBuffSize)) {
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
LAC_INVALID_PARAM_LOG(
"Invalid session state, stateful sessions "
"are not supported");
return CPA_STATUS_UNSUPPORTED;
}
if (!(pService->generic_service_info.dcExtendedFeatures &
DC_CNV_EXTENDED_CAPABILITY)) {
LAC_INVALID_PARAM_LOG(
"CompressAndVerify feature not supported");
return CPA_STATUS_UNSUPPORTED;
}
if (!(pService->generic_service_info.dcExtendedFeatures &
DC_CNVNR_EXTENDED_CAPABILITY)) {
LAC_INVALID_PARAM_LOG(
"CompressAndVerifyAndRecovery feature not supported");
return CPA_STATUS_UNSUPPORTED;
}
return dcCompDecompData(pService,
pSessionDesc,
insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
NULL,
callbackTag,
DC_COMPRESSION_REQUEST,
CPA_TRUE,
DC_CNVNR);
}
CpaStatus
cpaDcCompressData2(CpaInstanceHandle dcInstance,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcOpData *pOpData,
CpaDcRqResults *pResults,
void *callbackTag)
{
sal_compression_service_t *pService = NULL;
dc_session_desc_t *pSessionDesc = NULL;
CpaInstanceHandle insHandle = NULL;
Cpa64U srcBuffSize = 0;
dc_cnv_mode_t cnvMode = DC_NO_CNV;
LAC_CHECK_NULL_PARAM(pOpData);
if (((CPA_TRUE != pOpData->compressAndVerify) &&
(CPA_FALSE != pOpData->compressAndVerify)) ||
((CPA_FALSE != pOpData->compressAndVerifyAndRecover) &&
(CPA_TRUE != pOpData->compressAndVerifyAndRecover))) {
return CPA_STATUS_INVALID_PARAM;
}
if ((CPA_FALSE == pOpData->compressAndVerify) &&
(CPA_TRUE == pOpData->compressAndVerifyAndRecover)) {
return CPA_STATUS_INVALID_PARAM;
}
if ((CPA_TRUE == pOpData->compressAndVerify) &&
(CPA_TRUE == pOpData->compressAndVerifyAndRecover) &&
(CPA_FALSE == pOpData->integrityCrcCheck)) {
return cpaDcCompressData(dcInstance,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
pOpData->flushFlag,
callbackTag);
}
if (CPA_FALSE == pOpData->compressAndVerify) {
LAC_INVALID_PARAM_LOG(
"Data compression without verification not allowed");
return CPA_STATUS_UNSUPPORTED;
}
if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
insHandle = dcGetFirstHandle();
} else {
insHandle = dcInstance;
}
pService = (sal_compression_service_t *)insHandle;
LAC_CHECK_NULL_PARAM(insHandle);
LAC_CHECK_NULL_PARAM(pSessionHandle);
LAC_CHECK_NULL_PARAM(pOpData);
/* Check if SAL is initialised otherwise return an error */
SAL_RUNNING_CHECK(insHandle);
/* This check is outside the parameter checking as it is needed to
* manage zero length requests */
if (LacBuffDesc_BufferListVerifyNull(pSrcBuff,
&srcBuffSize,
LAC_NO_ALIGNMENT_SHIFT) !=
CPA_STATUS_SUCCESS) {
LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter");
return CPA_STATUS_INVALID_PARAM;
}
/* Ensure this is a compression instance */
SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
if (CPA_TRUE == pOpData->compressAndVerify &&
CPA_DC_STATEFUL == pSessionDesc->sessState) {
LAC_INVALID_PARAM_LOG(
"Invalid session state, stateful sessions "
"not supported with CNV");
return CPA_STATUS_UNSUPPORTED;
}
if (!(pService->generic_service_info.dcExtendedFeatures &
DC_CNV_EXTENDED_CAPABILITY) &&
(CPA_TRUE == pOpData->compressAndVerify)) {
LAC_INVALID_PARAM_LOG(
"CompressAndVerify feature not supported");
return CPA_STATUS_UNSUPPORTED;
}
if (CPA_STATUS_SUCCESS !=
dcParamCheck(insHandle,
pSessionHandle,
pService,
pSrcBuff,
pDestBuff,
pResults,
pSessionDesc,
pOpData->flushFlag,
srcBuffSize)) {
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_STATUS_SUCCESS != dcCheckOpData(pService, pOpData)) {
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_TRUE != pOpData->compressAndVerify) {
if (srcBuffSize > DC_COMP_MAX_BUFF_SIZE) {
LAC_LOG_ERROR(
"Compression payload greater than 64KB is "
"unsupported, when CnV is disabled\n");
return CPA_STATUS_UNSUPPORTED;
}
}
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
/* Lock the session to check if there are in-flight stateful
* requests */
LAC_SPINLOCK(&(pSessionDesc->sessionLock));
/* Check if there is already one in-flight stateful request */
if (0 !=
qatUtilsAtomicGet(
&(pSessionDesc->pendingStatefulCbCount))) {
LAC_LOG_ERROR(
"Only one in-flight stateful request supported");
LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
return CPA_STATUS_RETRY;
}
if (0 == srcBuffSize) {
if (CPA_TRUE ==
dcZeroLengthRequests(pService,
pSessionDesc,
pResults,
pOpData->flushFlag,
callbackTag,
DC_COMPRESSION_REQUEST)) {
return CPA_STATUS_SUCCESS;
}
}
qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
}
if (CPA_TRUE == pOpData->compressAndVerify) {
cnvMode = DC_CNV;
}
return dcCompDecompData(pService,
pSessionDesc,
insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
pOpData->flushFlag,
pOpData,
callbackTag,
DC_COMPRESSION_REQUEST,
CPA_TRUE,
cnvMode);
}
static CpaStatus
dcDecompressDataCheck(CpaInstanceHandle insHandle,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcRqResults *pResults,
CpaDcFlush flushFlag,
Cpa64U *srcBufferSize)
{
sal_compression_service_t *pService = NULL;
dc_session_desc_t *pSessionDesc = NULL;
Cpa64U srcBuffSize = 0;
pService = (sal_compression_service_t *)insHandle;
LAC_CHECK_NULL_PARAM(insHandle);
/* Check if SAL is initialised otherwise return an error */
SAL_RUNNING_CHECK(insHandle);
/* This check is outside the parameter checking as it is needed to
* manage zero length requests */
if (LacBuffDesc_BufferListVerifyNull(pSrcBuff,
&srcBuffSize,
LAC_NO_ALIGNMENT_SHIFT) !=
CPA_STATUS_SUCCESS) {
LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter");
return CPA_STATUS_INVALID_PARAM;
}
/* Ensure this is a compression instance */
SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
if (dcCheckSourceData(pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
srcBuffSize,
NULL) != CPA_STATUS_SUCCESS) {
return CPA_STATUS_INVALID_PARAM;
}
if (dcCheckDestinationData(pService,
pSessionHandle,
pDestBuff,
DC_DECOMPRESSION_REQUEST) !=
CPA_STATUS_SUCCESS) {
return CPA_STATUS_INVALID_PARAM;
}
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
LAC_INVALID_PARAM_LOG("Invalid sessDirection value");
return CPA_STATUS_INVALID_PARAM;
}
*srcBufferSize = srcBuffSize;
return CPA_STATUS_SUCCESS;
}
CpaStatus
cpaDcDecompressData(CpaInstanceHandle dcInstance,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcRqResults *pResults,
CpaDcFlush flushFlag,
void *callbackTag)
{
sal_compression_service_t *pService = NULL;
dc_session_desc_t *pSessionDesc = NULL;
CpaInstanceHandle insHandle = NULL;
Cpa64U srcBuffSize = 0;
CpaStatus status = CPA_STATUS_SUCCESS;
if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
insHandle = dcGetFirstHandle();
} else {
insHandle = dcInstance;
}
status = dcDecompressDataCheck(insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
&srcBuffSize);
if (CPA_STATUS_SUCCESS != status) {
return status;
}
pService = (sal_compression_service_t *)insHandle;
/* Check if SAL is initialised otherwise return an error */
SAL_RUNNING_CHECK(insHandle);
/* This check is outside the parameter checking as it is needed to
* manage zero length requests */
if (CPA_STATUS_SUCCESS !=
LacBuffDesc_BufferListVerifyNull(pSrcBuff,
&srcBuffSize,
LAC_NO_ALIGNMENT_SHIFT)) {
QAT_UTILS_LOG("Invalid source buffer list parameter");
return CPA_STATUS_INVALID_PARAM;
}
/* Ensure this is a compression instance */
SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
if (dcCheckSourceData(pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
srcBuffSize,
NULL) != CPA_STATUS_SUCCESS) {
return CPA_STATUS_INVALID_PARAM;
}
if (dcCheckDestinationData(pService,
pSessionHandle,
pDestBuff,
DC_DECOMPRESSION_REQUEST) !=
CPA_STATUS_SUCCESS) {
return CPA_STATUS_INVALID_PARAM;
}
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
QAT_UTILS_LOG("Invalid sessDirection value");
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
/* Lock the session to check if there are in-flight stateful
* requests */
LAC_SPINLOCK(&(pSessionDesc->sessionLock));
/* Check if there is already one in-flight stateful request */
if (0 !=
qatUtilsAtomicGet(
&(pSessionDesc->pendingStatefulCbCount))) {
LAC_LOG_ERROR(
"Only one in-flight stateful request supported");
LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
return CPA_STATUS_RETRY;
}
/* Gen 4 handle 0 len requests in FW */
if (isDcGen2x(pService)) {
if ((0 == srcBuffSize) ||
((1 == srcBuffSize) &&
(CPA_DC_FLUSH_FINAL != flushFlag) &&
(CPA_DC_FLUSH_FULL != flushFlag))) {
if (CPA_TRUE ==
dcZeroLengthRequests(
pService,
pSessionDesc,
pResults,
flushFlag,
callbackTag,
DC_DECOMPRESSION_REQUEST)) {
return CPA_STATUS_SUCCESS;
}
}
}
qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
}
return dcCompDecompData(pService,
pSessionDesc,
insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
flushFlag,
NULL,
callbackTag,
DC_DECOMPRESSION_REQUEST,
CPA_TRUE,
DC_NO_CNV);
}
CpaStatus
cpaDcDecompressData2(CpaInstanceHandle dcInstance,
CpaDcSessionHandle pSessionHandle,
CpaBufferList *pSrcBuff,
CpaBufferList *pDestBuff,
CpaDcOpData *pOpData,
CpaDcRqResults *pResults,
void *callbackTag)
{
sal_compression_service_t *pService = NULL;
dc_session_desc_t *pSessionDesc = NULL;
CpaInstanceHandle insHandle = NULL;
CpaStatus status = CPA_STATUS_SUCCESS;
Cpa64U srcBuffSize = 0;
LAC_CHECK_NULL_PARAM(pOpData);
if (CPA_FALSE == pOpData->integrityCrcCheck) {
return cpaDcDecompressData(dcInstance,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
pOpData->flushFlag,
callbackTag);
}
if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
insHandle = dcGetFirstHandle();
} else {
insHandle = dcInstance;
}
status = dcDecompressDataCheck(insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
pOpData->flushFlag,
&srcBuffSize);
if (CPA_STATUS_SUCCESS != status) {
return status;
}
pService = (sal_compression_service_t *)insHandle;
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
LAC_CHECK_NULL_PARAM(insHandle);
/* Check if SAL is initialised otherwise return an error */
SAL_RUNNING_CHECK(insHandle);
/* This check is outside the parameter checking as it is needed to
* manage zero length requests */
if (CPA_STATUS_SUCCESS !=
LacBuffDesc_BufferListVerifyNull(pSrcBuff,
&srcBuffSize,
LAC_NO_ALIGNMENT_SHIFT)) {
QAT_UTILS_LOG("Invalid source buffer list parameter");
return CPA_STATUS_INVALID_PARAM;
}
/* Ensure this is a compression instance */
SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
if (CPA_STATUS_SUCCESS !=
dcCheckSourceData(pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
CPA_DC_FLUSH_NONE,
srcBuffSize,
NULL)) {
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_STATUS_SUCCESS !=
dcCheckDestinationData(pService,
pSessionHandle,
pDestBuff,
DC_DECOMPRESSION_REQUEST)) {
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_STATUS_SUCCESS != dcCheckOpData(pService, pOpData)) {
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
QAT_UTILS_LOG("Invalid sessDirection value");
return CPA_STATUS_INVALID_PARAM;
}
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
/* Lock the session to check if there are in-flight stateful
* requests */
LAC_SPINLOCK(&(pSessionDesc->sessionLock));
/* Check if there is already one in-flight stateful request */
if (0 !=
qatUtilsAtomicGet(
&(pSessionDesc->pendingStatefulCbCount))) {
LAC_LOG_ERROR(
"Only one in-flight stateful request supported");
LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
return CPA_STATUS_RETRY;
}
/* Gen 4 handle 0 len requests in FW */
if (isDcGen2x(pService)) {
if ((0 == srcBuffSize) ||
((1 == srcBuffSize) &&
(CPA_DC_FLUSH_FINAL != pOpData->flushFlag) &&
(CPA_DC_FLUSH_FULL != pOpData->flushFlag))) {
if (CPA_TRUE ==
dcZeroLengthRequests(
pService,
pSessionDesc,
pResults,
pOpData->flushFlag,
callbackTag,
DC_DECOMPRESSION_REQUEST)) {
return CPA_STATUS_SUCCESS;
}
}
}
qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
}
return dcCompDecompData(pService,
pSessionDesc,
insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
pResults,
pOpData->flushFlag,
pOpData,
callbackTag,
DC_DECOMPRESSION_REQUEST,
CPA_TRUE,
DC_NO_CNV);
}
Reference in a new issue View git blame Copy permalink