opt/annokill/AnnoKill.cpp (667 lines of code) (raw):

/* * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. */ #include "AnnoKill.h" #include "ClassHierarchy.h" #include "Debug.h" #include "DexAnnotation.h" #include "DexClass.h" #include "DexLoader.h" #include "DexOutput.h" #include "DexUtil.h" #include "PassManager.h" #include "Resolver.h" #include "Show.h" #include "Timer.h" #include "Walkers.h" constexpr const char* METRIC_ANNO_KILLED = "num_anno_killed"; constexpr const char* METRIC_ANNO_TOTAL = "num_anno_total"; constexpr const char* METRIC_CLASS_ASETS_CLEARED = "num_class_cleared"; constexpr const char* METRIC_CLASS_ASETS_TOTAL = "num_class_total"; constexpr const char* METRIC_METHOD_ASETS_CLEARED = "num_method_cleared"; constexpr const char* METRIC_METHOD_ASETS_TOTAL = "num_method_total"; constexpr const char* METRIC_METHODPARAM_ASETS_CLEARED = "num_methodparam_cleared"; constexpr const char* METRIC_METHODPARAM_ASETS_TOTAL = "num_methodparam_total"; constexpr const char* METRIC_FIELD_ASETS_CLEARED = "num_field_cleared"; constexpr const char* METRIC_FIELD_ASETS_TOTAL = "num_field_total"; constexpr const char* METRIC_SIGNATURES_KILLED = "num_signatures_killed"; AnnoKill::AnnoKill( Scope& scope, bool only_force_kill, bool kill_bad_signatures, const AnnoNames& keep, const AnnoNames& kill, const AnnoNames& force_kill, const std::unordered_map<std::string, std::vector<std::string>>& class_hierarchy_keep_annos, const std::unordered_map<std::string, std::vector<std::string>>& annotated_keep_annos) : m_scope(scope), m_only_force_kill(only_force_kill), m_kill_bad_signatures(kill_bad_signatures) { TRACE(ANNO, 2, "only_force_kill=%u kill_bad_signatures=%d", m_only_force_kill, kill_bad_signatures); // Load annotations that should not be deleted. TRACE(ANNO, 2, "Keep annotations count %zu", keep.size()); for (const auto& anno_name : keep) { auto anno_type = DexType::get_type(anno_name.c_str()); TRACE(ANNO, 2, "Keep annotation type string %s", anno_name.c_str()); if (anno_type) { TRACE(ANNO, 2, "Keep annotation type %s", SHOW(anno_type)); m_keep.insert(anno_type); } else { TRACE(ANNO, 2, "Cannot find annotation type %s", anno_name.c_str()); } } // Load annotations we know and want dead. for (auto const& anno_name : kill) { DexType* anno = DexType::get_type(anno_name.c_str()); TRACE(ANNO, 2, "Kill annotation type string %s", anno_name.c_str()); if (anno) { TRACE(ANNO, 2, "Kill anno: %s", SHOW(anno)); m_kill.insert(anno); } else { TRACE(ANNO, 2, "Cannot find annotation type %s", anno_name.c_str()); } } // Load annotations we know and want dead. for (auto const& anno_name : force_kill) { DexType* anno = DexType::get_type(anno_name.c_str()); TRACE(ANNO, 2, "Force kill annotation type string %s", anno_name.c_str()); if (anno) { TRACE(ANNO, 2, "Force kill anno: %s", SHOW(anno)); m_force_kill.insert(anno); } else { TRACE(ANNO, 2, "Cannot find annotation type %s", anno_name.c_str()); } } // Populate class hierarchy keep map auto ch = build_type_hierarchy(m_scope); for (const auto& it : class_hierarchy_keep_annos) { auto* type = DexType::get_type(it.first.c_str()); auto* type_cls = type ? type_class(type) : nullptr; if (type_cls == nullptr) { continue; } TypeSet type_refs; get_all_children_or_implementors(ch, m_scope, type_cls, type_refs); for (auto& anno : it.second) { auto* anno_type = DexType::get_type(anno.c_str()); for (auto type_ref : type_refs) { m_anno_class_hierarchy_keep[type_ref].insert(anno_type); } } } for (const auto& it : m_anno_class_hierarchy_keep) { for (auto type : it.second) { TRACE(ANNO, 4, "anno_class_hier_keep: %s -> %s", it.first->get_name()->c_str(), type->get_name()->c_str()); } } // Populate anno keep map for (const auto& it : annotated_keep_annos) { auto* type = DexType::get_type(it.first.c_str()); for (auto& anno : it.second) { auto* anno_type = DexType::get_type(anno.c_str()); m_annotated_keep_annos[type].insert(anno_type); } } } AnnoKill::AnnoSet AnnoKill::get_referenced_annos() { Timer timer{"get_referenced_annos"}; AnnoKill::AnnoSet all_annos; // all used annotations auto annos_in_aset = [&](DexAnnotationSet* aset) { if (!aset) { return; } for (const auto& anno : aset->get_annotations()) { all_annos.insert(anno->type()); } }; for (const auto& cls : m_scope) { // all annotations referenced in classes annos_in_aset(cls->get_anno_set()); // all classes marked as annotation if (is_annotation(cls)) { all_annos.insert(cls->get_type()); } } // all annotations in methods walk::methods(m_scope, [&](DexMethod* method) { annos_in_aset(method->get_anno_set()); auto param_annos = method->get_param_anno(); if (!param_annos) { return; } for (auto& pa : *param_annos) { annos_in_aset(pa.second.get()); } }); // all annotations in fields walk::fields(m_scope, [&](DexField* field) { annos_in_aset(field->get_anno_set()); }); AnnoKill::AnnoSet referenced_annos; // mark an annotation as "unremovable" if a field is typed with that // annotation walk::fields(m_scope, [&](DexField* field) { // don't look at fields defined on the annotation itself const auto field_cls_type = field->get_class(); if (all_annos.count(field_cls_type) > 0) { return; } const auto field_cls = type_class(field_cls_type); if (field_cls != nullptr && is_annotation(field_cls)) { return; } auto ftype = field->get_type(); if (all_annos.count(ftype) > 0) { TRACE(ANNO, 3, "Field typed with an annotation type %s.%s:%s", SHOW(field->get_class()), SHOW(field->get_name()), SHOW(ftype)); referenced_annos.insert(ftype); } }); // mark an annotation as "unremovable" if a method signature contains a type // with that annotation walk::methods(m_scope, [&](DexMethod* meth) { // don't look at methods defined on the annotation itself const auto meth_cls_type = meth->get_class(); if (all_annos.count(meth_cls_type) > 0) { return; } const auto meth_cls = type_class(meth_cls_type); if (meth_cls != nullptr && is_annotation(meth_cls)) { return; } const auto& has_anno = [&](DexType* type) { if (all_annos.count(type) > 0) { TRACE(ANNO, 3, "Method contains annotation type in signature %s.%s:%s", SHOW(meth->get_class()), SHOW(meth->get_name()), SHOW(meth->get_proto())); referenced_annos.insert(type); } }; const auto proto = meth->get_proto(); has_anno(proto->get_rtype()); for (const auto& arg : *proto->get_args()) { has_anno(arg); } }); ConcurrentSet<DexType*> concurrent_referenced_annos; auto add_concurrent_referenced_anno = [&](DexType* t) { if (!referenced_annos.count(t)) { concurrent_referenced_annos.insert(t); } }; // mark an annotation as "unremovable" if any opcode references the annotation // type walk::parallel::opcodes( m_scope, [](DexMethod*) { return true; }, [&add_concurrent_referenced_anno, &all_annos](DexMethod* meth, IRInstruction* insn) { // don't look at methods defined on the annotation itself const auto meth_cls_type = meth->get_class(); if (all_annos.count(meth_cls_type) > 0) { return; } const auto meth_cls = type_class(meth_cls_type); if (meth_cls != nullptr && is_annotation(meth_cls)) { return; } if (insn->has_type()) { auto type = insn->get_type(); if (all_annos.count(type) > 0) { add_concurrent_referenced_anno(type); TRACE(ANNO, 3, "Annotation referenced in type opcode\n\t%s.%s:%s - %s", SHOW(meth->get_class()), SHOW(meth->get_name()), SHOW(meth->get_proto()), SHOW(insn)); } } else if (insn->has_field()) { auto field = insn->get_field(); auto fdef = resolve_field(field, opcode::is_an_sfield_op(insn->opcode()) ? FieldSearch::Static : FieldSearch::Instance); if (fdef != nullptr) field = fdef; bool referenced = false; auto owner = field->get_class(); if (all_annos.count(owner) > 0) { referenced = true; add_concurrent_referenced_anno(owner); } auto type = field->get_type(); if (all_annos.count(type) > 0) { referenced = true; add_concurrent_referenced_anno(type); } if (referenced) { TRACE(ANNO, 3, "Annotation referenced in field opcode\n\t%s.%s:%s - %s", SHOW(meth->get_class()), SHOW(meth->get_name()), SHOW(meth->get_proto()), SHOW(insn)); } } else if (insn->has_method()) { auto method = insn->get_method(); DexMethod* methdef = resolve_method(method, opcode_to_search(insn), meth); if (methdef != nullptr) method = methdef; bool referenced = false; auto owner = method->get_class(); if (all_annos.count(owner) > 0) { referenced = true; add_concurrent_referenced_anno(owner); } auto proto = method->get_proto(); auto rtype = proto->get_rtype(); if (all_annos.count(rtype) > 0) { referenced = true; add_concurrent_referenced_anno(rtype); } auto arg_list = proto->get_args(); for (const auto& arg : *arg_list) { if (all_annos.count(arg) > 0) { referenced = true; add_concurrent_referenced_anno(arg); } } if (referenced) { TRACE(ANNO, 3, "Annotation referenced in method opcode\n\t%s.%s:%s - %s", SHOW(meth->get_class()), SHOW(meth->get_name()), SHOW(meth->get_proto()), SHOW(insn)); } } }); referenced_annos.insert(concurrent_referenced_annos.begin(), concurrent_referenced_annos.end()); return referenced_annos; } AnnoKill::AnnoSet AnnoKill::get_removable_annotation_instances() { // Determine which annotation classes are removable. std::unordered_set<DexType*> bannotations; for (auto clazz : m_scope) { if (!(clazz->get_access() & DexAccessFlags::ACC_ANNOTATION)) { continue; } auto* aset = clazz->get_anno_set(); if (!aset) { continue; } auto& annos = aset->get_annotations(); for (auto& anno : annos) { if (m_kill.count(anno->type())) { bannotations.insert(clazz->get_type()); TRACE( ANNO, 3, "removable annotation class %s", SHOW(clazz->get_type())); } } } return bannotations; } void AnnoKill::count_annotation(const DexAnnotation* da, AnnoKillStats& stats) const { auto inc_counter = [](auto&, auto& c, auto) { c++; }; if (da->system_visible()) { if (traceEnabled(ANNO, 3)) { m_system_anno_map.update(da->type()->get_name()->str(), inc_counter); } stats.visibility_system_count++; } else if (da->runtime_visible()) { if (traceEnabled(ANNO, 3)) { m_runtime_anno_map.update(da->type()->get_name()->str(), inc_counter); } stats.visibility_runtime_count++; } else if (da->build_visible()) { if (traceEnabled(ANNO, 3)) { m_build_anno_map.update(da->type()->get_name()->str(), inc_counter); } stats.visibility_build_count++; } } void AnnoKill::cleanup_aset( DexAnnotationSet* aset, const AnnoKill::AnnoSet& referenced_annos, AnnoKillStats& stats, const std::unordered_set<const DexType*>& keep_annos) const { stats.annotations += aset->size(); auto& annos = aset->get_annotations(); auto fn = [&](const auto& da) { auto anno_type = da->type(); count_annotation(da.get(), stats); if (referenced_annos.count(anno_type) > 0) { TRACE(ANNO, 3, "Annotation type %s with type referenced in " "code, skipping...\n\tannotation: %s", SHOW(anno_type), SHOW(da.get())); return false; } if (keep_annos.count(anno_type) > 0) { TRACE(ANNO, 4, "Prohibited from removing annotation %s", SHOW(da.get())); return false; } if (m_keep.count(anno_type) > 0) { TRACE(ANNO, 3, "Exclude annotation type %s, " "skipping...\n\tannotation: %s", SHOW(anno_type), SHOW(da.get())); return false; } if (m_kill.count(anno_type) > 0) { TRACE(ANNO, 3, "Annotation instance (type: %s) marked for removal, " "annotation: %s", SHOW(anno_type), SHOW(da.get())); stats.annotations_killed++; return true; } if (m_force_kill.count(anno_type) > 0) { TRACE(ANNO, 3, "Annotation instance (type: %s) marked for forced removal, " "annotation: %s", SHOW(anno_type), SHOW(da.get())); stats.annotations_killed++; return true; } if (!m_only_force_kill && !da->system_visible()) { TRACE(ANNO, 3, "Killing annotation instance %s", SHOW(da.get())); stats.annotations_killed++; return true; } if (anno_type == DexType::get_type("Ldalvik/annotation/Signature;")) { if (should_kill_bad_signature(da.get())) { stats.signatures_killed++; return true; } } return false; }; annos.erase(std::remove_if(annos.begin(), annos.end(), fn), annos.end()); } bool AnnoKill::should_kill_bad_signature(DexAnnotation* da) const { if (!m_kill_bad_signatures) return false; TRACE(ANNO, 3, "Examining @Signature instance %s", SHOW(da)); auto& elems = da->anno_elems(); for (auto& elem : elems) { auto& ev = elem.encoded_value; if (ev->evtype() != DEVT_ARRAY) continue; auto arrayev = static_cast<DexEncodedValueArray*>(ev.get()); auto const& evs = arrayev->evalues(); for (auto& strev : *evs) { if (strev->evtype() != DEVT_STRING) continue; const auto& sigstr = static_cast<DexEncodedValueString*>(strev.get())->string()->str(); always_assert(sigstr.length() > 0); const auto* sigcstr = sigstr.c_str(); // @Signature grammar is non-trivial[1], nevermind the fact that // Signatures are broken up into arbitrary arrays of strings concatenated // at runtime. It seems like types are reliably never broken apart, so we // can usually find an entire type name in each DexEncodedValueString. // // We also crudely approximate that something looks like a typename in the // first place since there's a lot of mark up in the @Signature grammar, // e.g. formal type parameter names. We look for things that look like // "L*/*", don't include ":" (formal type parameter separator), and may or // may not end with a semicolon or angle bracket. // // I'm working on a C++ port of the AOSP generic signature parser so we // can make this more robust in the future. // // [1] androidxref.com/8.0.0_r4/xref/libcore/luni/src/main/java/libcore/ // reflect/GenericSignatureParser.java if (sigstr[0] == 'L' && strchr(sigcstr, '/') && !strchr(sigcstr, ':')) { auto* sigtype = DexType::get_type(sigstr.c_str()); if (!sigtype) { // Try with semicolon. sigtype = DexType::get_type(sigstr + ';'); } if (!sigtype && sigstr.back() == '<') { // Try replacing angle bracket with semicolon // d8 often encodes signature annotations this way std::string copy = sigstr; copy.pop_back(); copy.push_back(';'); sigtype = DexType::get_type(copy); } if (sigtype) { auto* sigcls = type_class(sigtype); if (!sigcls) { sigtype = nullptr; } else if (!sigcls->is_external()) { bool found = false; for (auto cls : m_scope) { if (cls == sigcls) { // Valid class, we're good, go to element in array found = true; continue; } } // Could not find the (non-external) class in Scope, so set signal // to kill if (!found) { sigtype = nullptr; } } } if (!sigtype) { TRACE(ANNO, 3, "Killing bad @Signature: %s", sigcstr); return true; } } } } return false; } std::unordered_set<const DexType*> AnnoKill::build_anno_keep( DexAnnotationSet* aset) const { std::unordered_set<const DexType*> keep_list; for (const auto& anno : aset->get_annotations()) { auto it = m_annotated_keep_annos.find(anno->type()); if (it != m_annotated_keep_annos.end()) { keep_list.insert(it->second.begin(), it->second.end()); } } return keep_list; } bool AnnoKill::kill_annotations() { const auto& referenced_annos = get_referenced_annos(); if (!m_only_force_kill) { m_kill = get_removable_annotation_instances(); } { Timer timer{"optimize classes"}; m_stats += walk::parallel::classes<AnnoKillStats>(m_scope, [&](auto* clazz) { AnnoKillStats local_stats{}; DexAnnotationSet* aset = clazz->get_anno_set(); if (!aset) { return local_stats; } auto keep_list = build_anno_keep(aset); { auto it = m_anno_class_hierarchy_keep.find(clazz->get_type()); if (it != m_anno_class_hierarchy_keep.end()) { keep_list.insert(it->second.begin(), it->second.end()); } } local_stats.class_asets++; cleanup_aset(aset, referenced_annos, local_stats, keep_list); if (aset->size() == 0) { TRACE(ANNO, 3, "Clearing annotation for class %s", SHOW(clazz->get_type())); clazz->clear_annotations(); local_stats.class_asets_cleared++; } return local_stats; }); } { Timer timer{"optimize methods"}; m_stats += walk::parallel::methods< AnnoKillStats>(m_scope, [&](DexMethod* method) { // Method annotations AnnoKillStats local_stats{}; auto method_aset = method->get_anno_set(); if (method_aset) { local_stats.method_asets++; auto keep_list = build_anno_keep(method_aset); cleanup_aset(method_aset, referenced_annos, local_stats, keep_list); if (method_aset->size() == 0) { TRACE(ANNO, 3, "Clearing annotations for method %s.%s:%s", SHOW(method->get_class()), SHOW(method->get_name()), SHOW(method->get_proto())); method->clear_annotations(); local_stats.method_asets_cleared++; } } // Parameter annotations. auto param_annos = method->get_param_anno(); if (param_annos) { local_stats.method_param_asets += param_annos->size(); bool clear_pas = true; for (auto& pa : *param_annos) { auto& param_aset = pa.second; if (param_aset->size() == 0) { continue; } auto keep_list = build_anno_keep(param_aset.get()); cleanup_aset( param_aset.get(), referenced_annos, local_stats, keep_list); if (param_aset->size() == 0) { continue; } clear_pas = false; } if (clear_pas) { TRACE(ANNO, 3, "Clearing parameter annotations for method parameters %s.%s:%s", SHOW(method->get_class()), SHOW(method->get_name()), SHOW(method->get_proto())); local_stats.method_param_asets_cleared += param_annos->size(); method->release_param_anno(); } } return local_stats; }); } { Timer timer{"optimize fields"}; m_stats += walk::parallel::fields<AnnoKillStats>(m_scope, [&](DexField* field) { AnnoKillStats local_stats{}; DexAnnotationSet* aset = field->get_anno_set(); if (!aset) { return local_stats; } local_stats.field_asets++; auto keep_list = build_anno_keep(aset); cleanup_aset(aset, referenced_annos, local_stats, keep_list); if (aset->size() == 0) { TRACE(ANNO, 3, "Clearing annotations for field %s.%s:%s", SHOW(field->get_class()), SHOW(field->get_name()), SHOW(field->get_type())); field->clear_annotations(); local_stats.field_asets_cleared++; } return local_stats; }); } bool classes_removed = false; // We're done removing annotation instances, go ahead and remove annotation // classes. m_scope.erase( std::remove_if(m_scope.begin(), m_scope.end(), [&](DexClass* cls) { if (!is_annotation(cls)) { return false; } auto type = cls->get_type(); if (referenced_annos.count(type)) { return false; } if (m_keep.count(type)) { return false; } TRACE( ANNO, 3, "Removing annotation type: %s", SHOW(type)); classes_removed = true; return true; }), m_scope.end()); if (traceEnabled(ANNO, 3)) { for (const auto& p : m_build_anno_map) { TRACE(ANNO, 3, "Build anno: %lu, %s", p.second, p.first.c_str()); } for (const auto& p : m_runtime_anno_map) { TRACE(ANNO, 3, "Runtime anno: %lu, %s", p.second, p.first.c_str()); } for (const auto& p : m_system_anno_map) { TRACE(ANNO, 3, "System anno: %lu, %s", p.second, p.first.c_str()); } } return classes_removed; } void AnnoKillPass::run_pass(DexStoresVector& stores, ConfigFiles&, PassManager& mgr) { auto scope = build_class_scope(stores); AnnoKill ak(scope, only_force_kill(), m_kill_bad_signatures, m_keep_annos, m_kill_annos, m_force_kill_annos, m_class_hierarchy_keep_annos, m_annotated_keep_annos); bool classes_removed = ak.kill_annotations(); if (classes_removed) { post_dexen_changes(scope, stores); } auto stats = ak.get_stats(); TRACE(ANNO, 1, "AnnoKill report killed/total"); TRACE(ANNO, 1, "Annotations: %zu/%zu", stats.annotations_killed, stats.annotations); TRACE(ANNO, 1, "Class Asets: %zu/%zu", stats.class_asets_cleared, stats.class_asets); TRACE(ANNO, 1, "Method Asets: %zu/%zu", stats.method_asets_cleared, stats.method_asets); TRACE(ANNO, 1, "MethodParam Asets: %zu/%zu", stats.method_param_asets_cleared, stats.method_param_asets); TRACE(ANNO, 1, "Field Asets: %zu/%zu", stats.field_asets_cleared, stats.field_asets); TRACE(ANNO, 3, "Total referenced Build Annos: %zu", stats.visibility_build_count); TRACE(ANNO, 3, "Total referenced Runtime Annos: %zu", stats.visibility_runtime_count); TRACE(ANNO, 3, "Total referenced System Annos: %zu", stats.visibility_system_count); TRACE(ANNO, 1, "@Signatures Killed: %zu", stats.signatures_killed); mgr.incr_metric(METRIC_ANNO_KILLED, stats.annotations_killed); mgr.incr_metric(METRIC_ANNO_TOTAL, stats.annotations); mgr.incr_metric(METRIC_CLASS_ASETS_CLEARED, stats.class_asets_cleared); mgr.incr_metric(METRIC_CLASS_ASETS_TOTAL, stats.class_asets); mgr.incr_metric(METRIC_METHOD_ASETS_CLEARED, stats.method_asets_cleared); mgr.incr_metric(METRIC_METHOD_ASETS_TOTAL, stats.method_asets); mgr.incr_metric(METRIC_METHODPARAM_ASETS_CLEARED, stats.method_param_asets_cleared); mgr.incr_metric(METRIC_METHODPARAM_ASETS_TOTAL, stats.method_param_asets); mgr.incr_metric(METRIC_FIELD_ASETS_CLEARED, stats.field_asets_cleared); mgr.incr_metric(METRIC_FIELD_ASETS_TOTAL, stats.field_asets); mgr.incr_metric(METRIC_SIGNATURES_KILLED, stats.signatures_killed); } static AnnoKillPass s_pass;