in src/rsa_support.c [171:211]
int OAEP_ENCODE(int sha,const octet *m,csprng *RNG,const octet *p,octet *f)
{
int slen;
int olen=f->max-1;
int mlen=m->len;
int hlen;
int seedlen;
char dbmask[MAX_RSA_BYTES];
char seed[64];
octet DBMASK= {0,sizeof(dbmask),dbmask};
octet SEED= {0,sizeof(seed),seed};
hlen=seedlen=sha;
if (mlen>olen-hlen-seedlen-1) return 1;
if (m==f) return 1; /* must be distinct octets */
hashit(sha,p,-1,f);
slen=olen-mlen-hlen-seedlen-1;
OCT_jbyte(f,0,slen);
OCT_jbyte(f,0x1,1);
OCT_joctet(f,m);
OCT_rand(&SEED,RNG,seedlen);
MGF1(sha,&SEED,olen-seedlen,&DBMASK);
OCT_xor(&DBMASK,f);
MGF1(sha,&DBMASK,seedlen,f);
OCT_xor(f,&SEED);
OCT_joctet(f,&DBMASK);
OCT_pad(f,f->max);
OCT_clear(&SEED);
OCT_clear(&DBMASK);
return 0;
}