in moonlight/training/generation/generation.py [0:0]
def process(self, item):
png_contents, staff_message = item
staff_message = musicscore_pb2.Staff.FromString(staff_message)
with tf.Session(graph=self.omr.graph) as sess:
# Load the image, then feed it in to apply noise.
# Randomly rotate the image and apply noise, then dump it back out as a
# PNG.
# TODO(ringw): Expose a way to pass in the image contents to the main
# OMR TF graph.
img = tf.to_float(tf.image.decode_png(png_contents))
# Collapse the RGB channels, if any. No-op for a monochrome PNG.
img = tf.reduce_mean(img[:, :, :3], axis=2)[:, :, None]
# Fix the stafflines being #999.
img = tf.clip_by_value(img * 2. - 255., 0., 255.)
img = self.noise_fn(img)
# Get a 2D uint8 image array for OMR.
noisy_image = sess.run(
tf.cast(tf.clip_by_value(img, 0, 255)[:, :, 0], tf.uint8))
# Run OMR staffline extraction and staffline distance estimation. The
# stafflines are used to get patches from the generated image.
stafflines, image_staffline_distance = sess.run(
[
self.omr.glyph_classifier.staffline_extractor.extract_staves(),
self.omr.structure.staff_detector.staffline_distance[0]
],
feed_dict={self.omr.image: noisy_image})
if stafflines.shape[0] != 1:
raise ValueError('Image should have one detected staff, got shape: ' +
str(stafflines.shape))
positive_example_count = 0
negative_example_whitelist = np.ones(
(stafflines.shape[staffline_extractor.Axes.POSITION],
stafflines.shape[staffline_extractor.Axes.X]), np.bool)
# Blacklist xs where the patch would overlap with either end.
negative_example_overlap_from_end = max(self.negative_example_distance,
self.patch_width // 2)
negative_example_whitelist[:, :negative_example_overlap_from_end] = False
negative_example_whitelist[:,
-negative_example_overlap_from_end - 1:] = False
all_positive_examples = []
for glyph in staff_message.glyph:
staffline = staffline_extractor.get_staffline(glyph.y_position,
stafflines[0])
glyph_x = int(
round(glyph.x *
self.omr.glyph_classifier.staffline_extractor.target_height /
(image_staffline_distance * self.omr.glyph_classifier
.staffline_extractor.staffline_distance_multiple)))
example = self._create_example(staffline, glyph_x, glyph.type)
if example:
staffline_index = staffline_extractor.y_position_to_index(
glyph.y_position,
stafflines.shape[staffline_extractor.Axes.POSITION])
# Blacklist the area adjacent to the glyph, even if it is not selected
# as a positive example below.
negative_example_whitelist[staffline_index, glyph_x -
self.negative_example_distance + 1:glyph_x +
self.negative_example_distance] = False
all_positive_examples.append(example)
positive_example_count += 1
for example in random.sample(all_positive_examples,
POSITIVE_EXAMPLES_PER_IMAGE):
yield example
self.patch_counter.inc()
negative_example_staffline, negative_example_x = np.where(
negative_example_whitelist)
negative_example_inds = np.random.choice(
len(negative_example_staffline),
int(positive_example_count * self.negative_to_positive_example_ratio))
negative_example_staffline = negative_example_staffline[
negative_example_inds]
negative_example_x = negative_example_x[negative_example_inds]
for staffline, x in zip(negative_example_staffline, negative_example_x):
example = self._create_example(stafflines[0, staffline], x,
musicscore_pb2.Glyph.NONE)
assert example, 'Negative example xs should always be in range'
yield example
self.patch_counter.inc()