in ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java [456:705]
private long getUsers(boolean computeDeletes) throws Throwable {
NamingEnumeration<SearchResult> userSearchResultEnum = null;
NamingEnumeration<SearchResult> groupSearchResultEnum = null;
long highestdeltaSyncUserTime;
try {
createLdapContext();
int total;
// Activate paged results
if (pagedResultsEnabled) {
ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)});
}
DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
if (!groupUserTable.rowKeySet().isEmpty() || !config.isDeltaSyncEnabled() || (computeDeletes)) {
// Fix RANGER-1957: Perform full sync when there are updates to the groups or when incremental sync is not enabled
deltaSyncUserTime = 0;
deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
}
extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))";
if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) {
String customFilter = userSearchFilter.trim();
if (!customFilter.startsWith("(")) {
customFilter = "(" + customFilter + ")";
}
extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")";
} else {
extendedUserSearchFilter = "(&" + extendedUserSearchFilter + ")";
}
LOG.info("extendedUserSearchFilter = {}", extendedUserSearchFilter);
highestdeltaSyncUserTime = deltaSyncUserTime;
// When multiple OUs are configured, go through each OU as the user search base to search for users.
for (String s : userSearchBase) {
byte[] cookie = null;
int counter = 0;
try {
int paged = 0;
do {
userSearchResultEnum = ldapContext.search(s, extendedUserSearchFilter, userSearchControls);
while (userSearchResultEnum.hasMore()) {
// searchResults contains all the user entries
final SearchResult userEntry = userSearchResultEnum.next();
if (userEntry == null) {
LOG.info("userEntry null, skipping sync for the entry");
continue;
}
Attributes attributes = userEntry.getAttributes();
if (attributes == null) {
LOG.info("attributes missing for entry {}, skipping sync", userEntry.getNameInNamespace());
continue;
}
Attribute userNameAttr = attributes.get(userNameAttribute);
if (userNameAttr == null) {
LOG.info("{} missing for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace());
continue;
}
String userFullName = (userEntry.getNameInNamespace());
String userName = (String) userNameAttr.get();
if (userName == null || userName.trim().isEmpty()) {
LOG.info("{} empty for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace());
continue;
}
Attribute timeStampAttr = attributes.get("uSNChanged");
if (timeStampAttr != null) {
String uSNChangedVal = (String) timeStampAttr.get();
long currentDeltaSyncTime = Long.parseLong(uSNChangedVal);
LOG.info("uSNChangedVal = {} and currentDeltaSyncTime = {}", uSNChangedVal, currentDeltaSyncTime);
if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
highestdeltaSyncUserTime = currentDeltaSyncTime;
}
} else {
timeStampAttr = attributes.get("modifytimestamp");
if (timeStampAttr != null) {
String timeStampVal = (String) timeStampAttr.get();
Date parseDate = dateFormat.parse(timeStampVal);
long currentDeltaSyncTime = parseDate.getTime();
LOG.info("timeStampVal = {} and currentDeltaSyncTime = {}", timeStampVal, currentDeltaSyncTime);
if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
highestdeltaSyncUserTime = currentDeltaSyncTime;
deltaSyncUserTimeStamp = timeStampVal;
}
}
}
// Get all the groups from the group name attribute of the user only when group search is not enabled.
if (!groupSearchEnabled) {
for (String useGroupNameAttribute : userGroupNameAttributeSet) {
Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
if (userGroupfAttribute != null) {
NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
while (groupEnum.hasMore()) {
String groupDN = (String) groupEnum.next();
LOG.debug("Adding {} to {}", groupDN, userName);
Map<String, String> groupAttrMap = new HashMap<>();
String groupName = getShortName(groupDN);
groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName);
groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupDN);
groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
sourceGroups.put(groupDN, groupAttrMap);
LOG.debug("As groupsearch is disabled, adding group {} from user memberof attribute for user {}", groupName, userName);
groupUserTable.put(groupDN, userFullName, userFullName);
}
}
}
}
Map<String, String> userAttrMap = new HashMap<>();
userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userFullName);
userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
Attribute userCloudIdAttr = attributes.get(userCloudIdAttribute);
if (userCloudIdAttr != null) {
addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr, config.getUserCloudIdAttributeDataType());
}
for (String otherUserAttribute : otherUserAttributes) {
if (attributes.get(otherUserAttribute) != null) {
String attrType = config.getOtherUserAttributeDataType(otherUserAttribute);
addToAttrMap(userAttrMap, otherUserAttribute, attributes.get(otherUserAttribute), attrType);
}
}
sourceUsers.put(userFullName, userAttrMap);
if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName))) {
Map<String, String> userMap = groupUserTable.column(userFullName);
if (MapUtils.isEmpty(userMap)) {
userMap = groupUserTable.column(userName);
}
for (Map.Entry<String, String> entry : userMap.entrySet()) {
LOG.debug("Updating groupUserTable {} with: {} for {}", entry.getValue(), userName, entry.getKey());
groupUserTable.put(entry.getKey(), userFullName, userFullName);
}
}
counter++;
if (counter <= 2000) {
LOG.info("Updating user count: {}, userName: {}", counter, userName);
if (counter == 2000) {
LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <===");
}
} else {
if (LOG.isTraceEnabled()) {
LOG.trace("Updating user count: {}, userName: {}", counter, userName);
} else if (counter % 100 == 0) {
LOG.info("Synced {} users till now", counter);
}
}
}
// Examine the paged results control response
Control[] controls = ldapContext.getResponseControls();
if (controls != null) {
for (Control control : controls) {
if (control instanceof PagedResultsResponseControl) {
PagedResultsResponseControl prrc = (PagedResultsResponseControl) control;
total = prrc.getResultSize();
if (total != 0) {
LOG.debug("END-OF-PAGE total : {}", total);
} else {
LOG.debug("END-OF-PAGE total : unknown");
}
cookie = prrc.getCookie();
}
}
} else {
LOG.debug("No controls were sent from the server");
}
// Re-activate paged results
if (pagedResultsEnabled) {
LOG.debug("Fetched paged results round: {}", ++paged);
ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)});
}
}
while (cookie != null);
LOG.info("LdapUserGroupBuilder.getUsers() completed with user count: {}", counter);
} catch (Exception t) {
LOG.error("LdapUserGroupBuilder.getUsers() failed with exception: ", t);
LOG.info("LdapUserGroupBuilder.getUsers() user count: {}", counter);
}
}
} finally {
if (userSearchResultEnum != null) {
userSearchResultEnum.close();
}
if (groupSearchResultEnum != null) {
groupSearchResultEnum.close();
}
closeLdapContext();
}
LOG.debug("highestDeltaSyncUserTime = {}", highestdeltaSyncUserTime);
return highestdeltaSyncUserTime;
}