public static void getIncremental()

in api-reference-examples/java/te-tag-query/com/facebook/threatexchange/Net.java [363:520]
105 lines of code
12 McCabe index (conditional complexity)

  public static void getIncremental(
    String tagName,
    String td_indicator_type,
    String since,
    int pageSize,
    boolean verbose,
    boolean showURLs,
    DescriptorFormatter descriptorFormatter,
    boolean includeIndicatorInOutput
  ) {
    // See also
    // https://developers.facebook.com/docs/threat-exchange/reference/apis/threat-descriptor/v6.0
    // for available fields
    String pageLimit = Integer.toString(pageSize);
    String startURL = TE_BASE_URL
      + "/threat_descriptors"
      + "/?access_token=" + APP_TOKEN
      + "&fields=raw_indicator,type,added_on,last_updated,first_active,last_active,expired_on,confidence,owner,review_status,privacy_type,status,severity,share_level,tags,description"
      + "&limit=" + pageLimit
      + "&tags=" + tagName
      + "&since=" + since;
    if (td_indicator_type != null) {
      startURL = startURL + "&type=" + td_indicator_type;
    }

    String nextURL = startURL;

    int pageIndex = 0;
    do {
      if (showURLs) {
        System.out.println("URL:");
        System.out.println(nextURL);
      }
      try (InputStream response = new URL(nextURL).openConnection().getInputStream()) {

        // {
        //    "data": [
        //     {
        //        "added_on": "2018-02-15T10:01:38+0000",
        //        "last_updated": "2018-02-15T10:01:38+0000",
        //        "confidence": 50,
        //        "description": "Description goes here",
        //        "id": "9998888887828886",
        //        "indicator": {
        //           "id": "8858889164495553",
        //           "indicator": "0096f7ffffb07f385630008f495b59ff",
        //           "type": "HASH_MD5"
        //        },
        //        "last_updated": "2018-02-15T10:01:39+0000",
        //        "owner": {
        //           "id": "9977777020662433",
        //           "email": "username\u0040companyname.com",
        //           "name": "Name of App"
        //        },
        //        "precision": "UNKNOWN",
        //        "privacy_type": "HAS_PRIVACY_GROUP",
        //        "raw_indicator": "0096f7ffffb07f385630008f495b59ff",
        //        "review_status": "REVIEWED_MANUALLY",
        //        "severity": "WARNING",
        //        "share_level": "AMBER",
        //        "status": "MALICIOUS",
        //        "type": "HASH_MD5"
        //     },
        //    "paging": {
        //       "cursors": {
        //          "before": "MAZDZD",
        //          "after": "MQZDZD"
        //       }
        //    }
        // }

        JSONObject object = (JSONObject) new JSONParser().parse(new InputStreamReader(response));
        JSONArray data = (JSONArray) object.get("data");
        JSONObject paging = (JSONObject) object.get("paging");
        if (paging == null) {
          nextURL = null;
        } else {
          nextURL = (String) paging.get("next");
        }

        int numItems = data.size();
        if (verbose) {
          SimpleJSONWriter w = new SimpleJSONWriter();
          w.add("page_index", pageIndex);
          w.add("num_items", numItems);
          System.out.println(w.format());
          System.out.flush();
        }

        for (int i = 0; i < numItems; i++) {
          JSONObject item = (JSONObject) data.get(i);

          String itemID = (String) item.get("id");
          String itemType = (String) item.get("type");
          String itemText = (String) item.get("raw_indicator");

          if (verbose) {
            SimpleJSONWriter w = new SimpleJSONWriter();
            w.add("id", itemID);
            w.add("type", itemType);
            w.add("indicator", itemText);
            System.out.println(w.format());
            System.out.flush();
          }

          JSONObject owner = (JSONObject)item.get("owner");

          List<String> tagTexts = new ArrayList<String>();
          JSONObject td_subjective_tags = (JSONObject)item.get("tags");
          if (td_subjective_tags != null) {
            JSONArray tag_data = (JSONArray)td_subjective_tags.get("data");
            int n = tag_data.size();
            for (int j = 0; j < n; j++) {
              JSONObject tag = (JSONObject) tag_data.get(j);
              String tagText = (String)tag.get("text");
              tagTexts.add(tagText);
            }
            Collections.sort(tagTexts); // canonicalize
          }

          String description = (String)item.get("description");
          if (description == null) {
            description = "";
          }

          ThreatDescriptor threatDescriptor = new ThreatDescriptor(
            itemID,
            itemText,
            itemType,
            (String)item.get("added_on"),
            (String)item.get("last_updated"),
            (String)item.get("first_active"),
            (String)item.get("last_active"),
            (String)item.get("expired_on"),
            Long.toString((Long)item.get("confidence")),
            (String)owner.get("id"),
            (String)owner.get("email"),
            (String)owner.get("name"),
            (String)item.get("privacy_type"),
            (String)item.get("review_status"),
            (String)item.get("status"),
            (String)item.get("severity"),
            (String)item.get("share_level"),
            tagTexts,
            description
          );

          // To do: move the printing to the caller via a callback.
          System.out.println(descriptorFormatter.format(threatDescriptor, includeIndicatorInOutput));
        }

        pageIndex++;
      } catch (Exception e) {
        e.printStackTrace(System.err);
        System.exit(1);
      }
    } while (nextURL != null);
  }