How to resolve the "EVP_DecryptFInal_ex: bad decrypt" during file decryption

Question

I have the following query.Could any one please suggest me a solution.

I'm working on encryption and decryption of file for first time.

I have encrypted file through command prompt using the command:

openssl enc -aes-256-cbc -in file.txt -out file.enc -k "key value" -iv "iv value"

I have to decrypt it programmatically. So I have written the program for it, but it is throwing the following error:

./exe_file enc_file_directory
...
error: 06065064: digital envelope routines: EVP_DecryptFInal_ex: bad decrypt: evp_enc.c

---

The program below takes input as directory path and search for encrypted file ".enc" and try to decrypt it read into buffer.

Code:

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <openssl/evp.h>
#include <openssl/err.h>
#include <openssl/conf.h>
#include <libxml/globals.h>

void handleErrors(char *msg)
{
    {
        ERR_print_errors_fp(stderr);
        printf("%s", msg);
        abort(); 
    }
}

void freeMemory(char *mem)
{
    if (NULL != mem)
    {
        free(mem);
        mem = NULL;
    }
}

/* Function to decrypt the XML files */

int decryptXML(unsigned char *indata, unsigned char *outdata, int fsize)
{

    int outlen1 = 0, outlen2 = 0;

    unsigned char iv[] = "b63e541bc9ece19a1339df4f8720dcc3";
    unsigned char ckey[] = "70bbc518c57acca2c2001694648c40ddaf19e3b4fe1376ad656de8887a0a5ec2" ;

    if (NULL == indata)
    {
        printf ("input data is empty\n");
        return 0;
    }

    if (0 >= fsize)
    {
        printf ("file size is zero\n");
        return 0;
    }

    outdata = (char *) malloc (sizeof (char) * fsize * 2);

    EVP_CIPHER_CTX ctx;

    EVP_CIPHER_CTX_init(&ctx);

    if (! EVP_DecryptInit_ex (&ctx, EVP_aes_256_cbc(), NULL, ckey, iv))
    {
        EVP_CIPHER_CTX_cleanup(&ctx);
    handleErrors("DInit");
    }

    if (! EVP_DecryptUpdate (&ctx, outdata, &outlen1, indata, fsize))
    {
        EVP_CIPHER_CTX_cleanup(&ctx);
        handleErrors("DUpdate");
    }

    if (! EVP_DecryptFinal_ex (&ctx, outdata + outlen1, &outlen2))
    {

        EVP_CIPHER_CTX_cleanup(&ctx);
        handleErrors("DFinal");
    }

    EVP_CIPHER_CTX_cleanup(&ctx);

    return outlen1+outlen2;

}

int isDirectory(char *path)
{
    DIR *dir = NULL;
    FILE *fin = NULL, *fout = NULL;
    int enc_len = 0, dec_len = 0, fsize = 0, ksize = 0;
    unsigned char *indata = NULL, *outdata = NULL;
    char buff[BUFFER_SIZE], file_path[BUFFER_SIZE], cur_dir[BUFFER_SIZE];

    struct dirent *in_dir;
    struct stat s;

    if (NULL == (dir = opendir(path)))
    {
        printf ("ERROR: Failed to open the directory %s\n", path);
        perror("cannot open.");
        exit(1);
    }

    while (NULL != (in_dir = readdir(dir)))
    {

        if (!strcmp (in_dir->d_name, ".") || !strcmp(in_dir->d_name, ".."))
            continue;

        sprintf (buff, "%s/%s", path, in_dir->d_name);

        if (-1 == stat(buff, &s))
        {
            perror("stat");
            exit(1);
        }

        if (S_ISDIR(s.st_mode))
        {

            isDirectory(buff);
        }
        else
        {
            strcpy(file_path, buff);

            if (strstr(file_path, ".enc"))
            {

                /* File to be decrypted */

                fout = fopen(file_path,"rb"); 

                fseek (fout, 0L, SEEK_END);
                fsize = ftell(fout);
                fseek (fout, 0L, SEEK_SET);

                indata = (char*)malloc(fsize);

                fread (indata, sizeof(char), fsize, fout);

                if (NULL == fout)
                {
                    perror("Cannot open enc file: ");
                    return 1;
                }


                dec_len = decryptXML (indata, outdata, fsize);
                outdata[dec_len] = '\0';
                printf ("%s\n", outdata);
                fclose (fin);
                fclose (fout);

            }
        }
    }



    closedir(dir);
    freeMemory(outdata);
    freeMemory(indata);

    return 1; 
}


int main(int argc, char *argv[])
{
    int result;

    if (argc != 2)
    {
        printf ("Usage: <executable> path_of_the_files\n");
        return -1;
    }

    ERR_load_crypto_strings();
    OpenSSL_add_all_algorithms();
    OPENSSL_config(NULL);

    /* Checking for the directory existance */

    result = isDirectory(argv[1]);

    EVP_cleanup();
    ERR_free_strings();

    if (0 == result)
        return 1;
    else
       return 0;
}

Thank you.

Answer 1

This message digital envelope routines: EVP_DecryptFInal_ex: bad decrypt can also occur when you encrypt and decrypt with an incompatible versions of openssl.

The issue I was having was that I was encrypting on Windows which had version 1.1.0 and then decrypting on a generic Linux system which had 1.0.2g.

It is not a very helpful error message!

---

Working solution:

A possible solution from @AndrewSavinykh that worked for many (see the comments):

Default digest has changed between those versions from md5 to sha256. One can specify the default digest on the command line as -md sha256 or -md md5 respectively