public int filterRow()

in src/main/java/org/apache/xmlgraphics/image/codec/png/PNGEncodeParam.java [1389:1530]


    public int filterRow(byte[] currRow,
                         byte[] prevRow,
                         byte[][] scratchRows,
                         int bytesPerRow,
                         int bytesPerPixel) {

        int [] badness = {0, 0, 0, 0, 0};
        int curr;
        int left;
        int up;
        int upleft;
        int diff;
        int pa;
        int pb;
        int pc;
        for (int i = bytesPerPixel; i < bytesPerRow + bytesPerPixel; i++) {
            curr   = currRow[i] & 0xff;
            left   = currRow[i - bytesPerPixel] & 0xff;
            up     = prevRow[i] & 0xff;
            upleft = prevRow[i - bytesPerPixel] & 0xff;

            // no filter
            badness[0] += curr;

            // sub filter
            diff = curr - left;
            scratchRows[1][i]  = (byte)diff;
            badness    [1]    +=   (diff > 0) ? diff : -diff;

            // up filter
            diff = curr - up;
            scratchRows[2][i]  = (byte)diff;
            badness    [2]    +=   (diff >= 0) ? diff : -diff;

            // average filter
            diff = curr - ((left + up) >> 1);
            scratchRows[3][i]  = (byte)diff;
            badness    [3]    +=   (diff >= 0) ? diff : -diff;

            // paeth filter

            // Original code much simplier but doesn't take full
            // advantage of relationship between pa/b/c and
            // information gleaned in abs operations.
            /// pa = up  -upleft;
            /// pb = left-upleft;
            /// pc = pa+pb;
            /// pa = abs(pa);
            /// pb = abs(pb);
            /// pc = abs(pc);
            /// if ((pa <= pb) && (pa <= pc))
            ///   diff = curr-left;
            /// else if (pb <= pc)
            ///   diff = curr-up;
            /// else
            ///   diff = curr-upleft;

            pa = up  - upleft;
            pb = left - upleft;
            if (pa < 0) {
              if (pb < 0) {
                // both pa & pb neg so pc is always greater than or
                // equal to pa or pb;
                if (pa >= pb) { // since pa & pb neg check sense is reversed.
                  diff = curr - left;
                } else {
                  diff = curr - up;
                }
              } else {
                // pa neg pb pos so we must compute pc...
                pc = pa + pb;
                pa = -pa;
                if (pa <= pb) { // pc is positive and less than pb
                  if (pa <= pc) {
                    diff = curr - left;
                  } else {
                    diff = curr - upleft;
                  }
                } else {
                  // pc is negative and less than or equal to pa,
                  // but since pa is greater than pb this isn't an issue...
                  if (pb <= -pc) {
                    diff = curr - up;
                  } else {
                    diff = curr - upleft;
                  }
                }
              }
            } else {
              if (pb < 0) {
                pb = -pb; // make it positive...
                if (pa <= pb) {
                  // pc would be negative and less than or equal to pb
                  pc = pb - pa;
                  if (pa <= pc) {
                    diff = curr - left;
                  } else if (pb == pc) {
                    // if pa is zero then pc==pb otherwise
                    // pc must be less than pb.
                    diff = curr - up;
                  } else {
                    diff = curr - upleft;
                  }
                } else {
                  // pc would be positive and less than pa.
                  pc = pa - pb;
                  if (pb <= pc) {
                    diff = curr - up;
                  } else {
                    diff = curr - upleft;
                  }
                }
              } else {
                // both pos so pa+pb is always greater than pa/pb
                if (pa <= pb) {
                  diff = curr - left;
                } else {
                  diff = curr - up;
                }
              }
            }
            scratchRows[4][i]  = (byte)diff;
            badness    [4]    +=   (diff >= 0) ? diff : -diff;
        }
        int filterType = 0;
        int minBadness = badness[0];

        for (int i = 1; i < 5; i++) {
            if (badness[i] < minBadness) {
                minBadness = badness[i];
                filterType = i;
            }
        }

        if (filterType == 0) {
            System.arraycopy(currRow, bytesPerPixel,
                             scratchRows[0], bytesPerPixel,
                             bytesPerRow);
        }

        return filterType;
    }