in src/main/java/org/apache/xmlgraphics/image/codec/png/PNGEncodeParam.java [1389:1530]
public int filterRow(byte[] currRow,
byte[] prevRow,
byte[][] scratchRows,
int bytesPerRow,
int bytesPerPixel) {
int [] badness = {0, 0, 0, 0, 0};
int curr;
int left;
int up;
int upleft;
int diff;
int pa;
int pb;
int pc;
for (int i = bytesPerPixel; i < bytesPerRow + bytesPerPixel; i++) {
curr = currRow[i] & 0xff;
left = currRow[i - bytesPerPixel] & 0xff;
up = prevRow[i] & 0xff;
upleft = prevRow[i - bytesPerPixel] & 0xff;
// no filter
badness[0] += curr;
// sub filter
diff = curr - left;
scratchRows[1][i] = (byte)diff;
badness [1] += (diff > 0) ? diff : -diff;
// up filter
diff = curr - up;
scratchRows[2][i] = (byte)diff;
badness [2] += (diff >= 0) ? diff : -diff;
// average filter
diff = curr - ((left + up) >> 1);
scratchRows[3][i] = (byte)diff;
badness [3] += (diff >= 0) ? diff : -diff;
// paeth filter
// Original code much simplier but doesn't take full
// advantage of relationship between pa/b/c and
// information gleaned in abs operations.
/// pa = up -upleft;
/// pb = left-upleft;
/// pc = pa+pb;
/// pa = abs(pa);
/// pb = abs(pb);
/// pc = abs(pc);
/// if ((pa <= pb) && (pa <= pc))
/// diff = curr-left;
/// else if (pb <= pc)
/// diff = curr-up;
/// else
/// diff = curr-upleft;
pa = up - upleft;
pb = left - upleft;
if (pa < 0) {
if (pb < 0) {
// both pa & pb neg so pc is always greater than or
// equal to pa or pb;
if (pa >= pb) { // since pa & pb neg check sense is reversed.
diff = curr - left;
} else {
diff = curr - up;
}
} else {
// pa neg pb pos so we must compute pc...
pc = pa + pb;
pa = -pa;
if (pa <= pb) { // pc is positive and less than pb
if (pa <= pc) {
diff = curr - left;
} else {
diff = curr - upleft;
}
} else {
// pc is negative and less than or equal to pa,
// but since pa is greater than pb this isn't an issue...
if (pb <= -pc) {
diff = curr - up;
} else {
diff = curr - upleft;
}
}
}
} else {
if (pb < 0) {
pb = -pb; // make it positive...
if (pa <= pb) {
// pc would be negative and less than or equal to pb
pc = pb - pa;
if (pa <= pc) {
diff = curr - left;
} else if (pb == pc) {
// if pa is zero then pc==pb otherwise
// pc must be less than pb.
diff = curr - up;
} else {
diff = curr - upleft;
}
} else {
// pc would be positive and less than pa.
pc = pa - pb;
if (pb <= pc) {
diff = curr - up;
} else {
diff = curr - upleft;
}
}
} else {
// both pos so pa+pb is always greater than pa/pb
if (pa <= pb) {
diff = curr - left;
} else {
diff = curr - up;
}
}
}
scratchRows[4][i] = (byte)diff;
badness [4] += (diff >= 0) ? diff : -diff;
}
int filterType = 0;
int minBadness = badness[0];
for (int i = 1; i < 5; i++) {
if (badness[i] < minBadness) {
minBadness = badness[i];
filterType = i;
}
}
if (filterType == 0) {
System.arraycopy(currRow, bytesPerPixel,
scratchRows[0], bytesPerPixel,
bytesPerRow);
}
return filterType;
}