samza-yarn/src/main/java/org/apache/samza/job/yarn/LocalizerResourceMapper.java [41:98]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
public class LocalizerResourceMapper {
  private static final Logger log = LoggerFactory.getLogger(LocalizerResourceMapper.class);

  private final YarnConfiguration yarnConfiguration; //yarn configurations
  private final LocalizerResourceConfig resourceConfig;
  private final Map<String, LocalResource> localResourceMap;

  public LocalizerResourceMapper(LocalizerResourceConfig resourceConfig, YarnConfiguration yarnConfiguration) {
    this.yarnConfiguration = yarnConfiguration;
    this.resourceConfig = resourceConfig;
    this.localResourceMap = buildResourceMapping();
  }

  private Map<String, LocalResource> buildResourceMapping() {
    ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

    List<String> resourceNames = resourceConfig.getResourceNames();
    for (String resourceName : resourceNames) {
      String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
      LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
      LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
      Path resourcePath = resourceConfig.getResourcePath(resourceName);

      LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

      localResourceMapBuilder.put(resourceLocalName, localResource);
      log.info("preparing local resource: {}", resourceLocalName);
    }

    return localResourceMapBuilder.build();
  }

  private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
    LocalResource localResource = Records.newRecord(LocalResource.class);
    URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
    try {
      FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

      if (null == resourceFileStatus) {
        throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
      }

      localResource.setResource(resourceUrl);
      log.info("setLocalizerResource for {}", resourceUrl);
      localResource.setSize(resourceFileStatus.getLen());
      localResource.setTimestamp(resourceFileStatus.getModificationTime());
      localResource.setType(resourceType);
      localResource.setVisibility(resourceVisibility);
      return localResource;
    } catch (IOException ioe) {
      log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
      throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
    }

  }

  public Map<String, LocalResource> getResourceMap() {
    return ImmutableMap.copyOf(localResourceMap);
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



samza-yarn3/src/main/java/org/apache/samza/job/yarn/LocalizerResourceMapper.java [41:98]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
public class LocalizerResourceMapper {
  private static final Logger log = LoggerFactory.getLogger(LocalizerResourceMapper.class);

  private final YarnConfiguration yarnConfiguration; //yarn configurations
  private final LocalizerResourceConfig resourceConfig;
  private final Map<String, LocalResource> localResourceMap;

  public LocalizerResourceMapper(LocalizerResourceConfig resourceConfig, YarnConfiguration yarnConfiguration) {
    this.yarnConfiguration = yarnConfiguration;
    this.resourceConfig = resourceConfig;
    this.localResourceMap = buildResourceMapping();
  }

  private Map<String, LocalResource> buildResourceMapping() {
    ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

    List<String> resourceNames = resourceConfig.getResourceNames();
    for (String resourceName : resourceNames) {
      String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
      LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
      LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
      Path resourcePath = resourceConfig.getResourcePath(resourceName);

      LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

      localResourceMapBuilder.put(resourceLocalName, localResource);
      log.info("preparing local resource: {}", resourceLocalName);
    }

    return localResourceMapBuilder.build();
  }

  private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
    LocalResource localResource = Records.newRecord(LocalResource.class);
    URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
    try {
      FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

      if (null == resourceFileStatus) {
        throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
      }

      localResource.setResource(resourceUrl);
      log.info("setLocalizerResource for {}", resourceUrl);
      localResource.setSize(resourceFileStatus.getLen());
      localResource.setTimestamp(resourceFileStatus.getModificationTime());
      localResource.setType(resourceType);
      localResource.setVisibility(resourceVisibility);
      return localResource;
    } catch (IOException ioe) {
      log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
      throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
    }

  }

  public Map<String, LocalResource> getResourceMap() {
    return ImmutableMap.copyOf(localResourceMap);
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -