public void prepareForDecryption()

in pdfbox/src/main/java/org/apache/pdfbox/pdmodel/encryption/StandardSecurityHandler.java [146:297]
132 lines of code
25 McCabe index (conditional complexity)

    public void prepareForDecryption(PDEncryption encryption, COSArray documentIDArray,
                                     DecryptionMaterial decryptionMaterial)
                                     throws IOException
    {
        if(!(decryptionMaterial instanceof StandardDecryptionMaterial))
        {
            throw new IOException("Decryption material is not compatible with the document");
        }
        
        // This is only used with security version 4 and 5.
        if (encryption.getVersion() >= REVISION_4) {
            setStreamFilterName(encryption.getStreamFilterName());
            setStringFilterName(encryption.getStringFilterName());
        }
        setDecryptMetadata(encryption.isEncryptMetaData());
        StandardDecryptionMaterial material = (StandardDecryptionMaterial)decryptionMaterial;

        String password = material.getPassword();
        if(password == null)
        {
            password = "";
        }

        int dicPermissions = encryption.getPermissions();
        int dicRevision = encryption.getRevision();
        int dicLength = encryption.getVersion() == 1 ? 5 : encryption.getLength() / 8;
        
        if (encryption.getVersion() == REVISION_4 || encryption.getVersion() == REVISION_5)
        {
            // detect whether AES encryption is used. This assumes that the encryption algo is 
            // stored in the PDCryptFilterDictionary
            // However, crypt filters are used only when V is 4 or 5.
            PDCryptFilterDictionary stdCryptFilterDictionary = encryption.getStdCryptFilterDictionary();
            if (stdCryptFilterDictionary != null)
            {
                COSName cryptFilterMethod = stdCryptFilterDictionary.getCryptFilterMethod();
                if (COSName.AESV2.equals(cryptFilterMethod))
                {
                    dicLength = 128 / 8;
                    setAES(true);
                    if (encryption.getCOSObject().containsKey(COSName.LENGTH))
                    {
                        // PDFBOX-5345
                        int newLength = encryption.getLength() / 8;
                        if (newLength < dicLength)
                        {
                            LOG.warn(
                                    "Using {} bytes key length instead of {} in AESV2 encryption?!",
                                    newLength, dicLength);
                            dicLength = newLength;
                        }
                    }
                }
                if (COSName.AESV3.equals(cryptFilterMethod))
                {
                    dicLength = 256 / 8;
                    setAES(true);
                    if (encryption.getCOSObject().containsKey(COSName.LENGTH))
                    {
                        // PDFBOX-5345
                        int newLength = encryption.getLength() / 8;
                        if (newLength < dicLength)
                        {
                            LOG.warn(
                                    "Using {} bytes key length instead of {} in AESV3 encryption?!",
                                    newLength, dicLength);
                            dicLength = newLength;
                        }
                    }
                }
            }
        }

        byte[] documentIDBytes = getDocumentIDBytes(documentIDArray);

        // we need to know whether the meta data was encrypted for password calculation
        boolean encryptMetadata = encryption.isEncryptMetaData();
        
        byte[] userKey = encryption.getUserKey();
        byte[] ownerKey = encryption.getOwnerKey();
        byte[] ue = null, oe = null;

        Charset passwordCharset = StandardCharsets.ISO_8859_1;
        if (dicRevision == REVISION_5 || dicRevision == REVISION_6)
        {
            passwordCharset = StandardCharsets.UTF_8;
            ue = encryption.getUserEncryptionKey();
            oe = encryption.getOwnerEncryptionKey();
        }

        if (dicRevision == REVISION_6)
        {
            password = SaslPrep.saslPrepQuery(password); // PDFBOX-4155
        }

        AccessPermission currentAccessPermission;

        byte[] encryptedKey;
        byte[] passwordBytes;
        boolean isOwnerPassword;
        if( isOwnerPassword(password.getBytes(passwordCharset), userKey, ownerKey,
                                 dicPermissions, documentIDBytes, dicRevision,
                                 dicLength, encryptMetadata) )
        {
            currentAccessPermission = AccessPermission.getOwnerAccessPermission();
            setCurrentAccessPermission(currentAccessPermission);
            
            if (dicRevision == REVISION_5 || dicRevision == REVISION_6)
            {
                passwordBytes = password.getBytes(passwordCharset);
            }
            else
            {
                passwordBytes = getUserPassword234(password.getBytes(passwordCharset),
                        ownerKey, dicRevision, dicLength );
            }
            isOwnerPassword = true;
        }
        else if( isUserPassword(password.getBytes(passwordCharset), userKey, ownerKey,
                           dicPermissions, documentIDBytes, dicRevision,
                           dicLength, encryptMetadata) )
        {
            currentAccessPermission = new AccessPermission(dicPermissions);
            currentAccessPermission.setReadOnly();
            setCurrentAccessPermission(currentAccessPermission);
            passwordBytes = password.getBytes(passwordCharset);
            isOwnerPassword = false;
        }
        else
        {
            throw new InvalidPasswordException("Cannot decrypt PDF, the password is incorrect");
        }
        encryptedKey = computeEncryptedKey(
            passwordBytes,
            ownerKey, userKey, oe, ue,
            dicPermissions,
            documentIDBytes,
            dicRevision,
            dicLength,
            encryptMetadata, isOwnerPassword);
        if (dicRevision == REVISION_4 && encryptedKey.length < 16)
        {
            LOG.info("PDFBOX-5955: padding RC4 key to length 16");
            encryptedKey = Arrays.copyOf(encryptedKey, 16);
        }
        setEncryptionKey(encryptedKey);

        if (dicRevision == REVISION_5 || dicRevision == REVISION_6)
        {
            validatePerms(encryption, dicPermissions, encryptMetadata);
        }
    }