in aios/sql/python/local_search_starter.py [0:0]
def initMember(self, options):
self.searcherReplica = options.searcherReplica
self.qrsQueue = options.qrsQueue
self.searcherQueue = options.searcherQueue
self.qrsThreadNum = options.qrsThreadNum
self.qrsQueueSize = options.qrsQueueSize
self.searcherThreadNum = options.searcherThreadNum
self.naviThreadNum = options.naviThreadNum
self.searcherQueueSize = options.searcherQueueSize
self.threadNumScaleFactor = options.threadNumScaleFactor
self.indexPath = options.indexPath
self.aggName = options.aggName
self.basicTuringBizNames = options.basicTuringBizNames
self.tabletInfos = options.tabletInfos
self.basicTuringPrefix = options.basicTuringPrefix
self.paraSearchWays = options.paraSearchWays
self.disableSql = options.disableSql
self.disableSqlWarmup = options.disableSqlWarmup
self.enableMultiPartition = options.enableMultiPartition
self.enableLocalAccess = options.enableLocalAccess
self.onlySql = options.onlySql
self.enableLocalCatalog = options.enableLocalCatalog
self.enableUpdateCatalog = options.enableUpdateCatalog
self.dailyrunMode = options.dailyrunMode
self.multiBiz = options.multiBiz
self.modelBiz = set(options.modelBiz.split(','))
self.localBizService = options.localBizService
self.kmonSinkAddress = options.kmonSinkAddress
self.specialCatalogList = options.specialCatalogList
self.zkRoot = options.zkRoot
self.mode = options.mode
self.leaderElectionStrategyType = options.leaderElectionStrategyType
self.leaderElectionConfig = options.leaderElectionConfig
self.versionSyncConfig = options.versionSyncConfig
self.disableCodeGen = options.disableCodeGen
self.disableTurbojet = options.disableTurbojet
self.searcherLocalSubscribe = options.searcherLocalSubscribe
self.enablePublishTableTopoInfo = options.enablePublishTableTopoInfo
self.forceTabletLoad = options.forceTabletLoad
self.allowFollowWrite = options.allowFollowWrite
if not self.indexPath.startswith('/'):
self.indexPath = os.path.join(os.getcwd(), self.indexPath)
self.configPath = options.configPath
if not self.configPath.startswith('/'):
self.configPath = os.path.join(os.getcwd(), self.configPath)
self.onlineConfigPath = os.path.join(self.configPath, "bizs")
self.offlineConfigPath = os.path.join(self.configPath, "table")
tableVersions = sorted(map(lambda x: int(x), os.listdir(self.offlineConfigPath)))
if len(tableVersions) == 0:
print "table version count is 0, path [%s]" % self.offlineConfigPath
else:
self.offlineConfigPath = os.path.join(self.offlineConfigPath, str(tableVersions[-1]))
self.qrsHttpArpcBindPort = options.qrsHttpArpcBindPort
self.qrsArpcBindPort = options.qrsArpcBindPort
self.portList = []
self.origin_port_list = map(lambda x: int(x), options.portList.split(","))
self.searcher_port_list = []
self.qrs_port_list = None
self.portStart = 12000
if len(self.portList) > 0:
self.portStart = int(self.portList[0])
else:
self.portList = [12000]
if options.zoneName:
self.specialZones = options.zoneName.split(",")
else:
self.specialZones = []
self.atableList = {}
for zone in self.specialZones:
self.atableList[zone] = ""
if options.atables:
azones = options.atables.split(";")
for azone in azones:
tmp = azone.split(":")
self.atableList[tmp[0]] = tmp[1]
if options.binaryPath:
self.binaryPath = options.binaryPath
if not options.binaryPath.startswith('/'):
self.binaryPath = os.path.join(os.getcwd(), options.binaryPath)
else:
curdir = os.path.split(os.path.realpath(__file__))[0]
self.binaryPath = os.path.join(curdir, "../../../../../../")
self.binaryPath = os.path.abspath(self.binaryPath)
self.timeout = options.timeout
self.preload = options.preload
self.serviceName = options.serviceName
self.amonPath = options.amonPath
self.libPath = "/usr/ali/java/jre/lib/amd64/server:/usr/local/java/jdk/jre/lib/amd64/server:" + self.binaryPath + "/home/admin/sap/lib64/:" + self.binaryPath + "/home/admin/diamond-client4cpp/lib:" + self.binaryPath + "/home/admin/eagleeye-core/lib/:" + self.binaryPath + "/usr/local/cuda-10.1/lib64/stubs:" + \
self.binaryPath + "/usr/local/cuda-10.1/lib64/:" + self.binaryPath + "/usr/local/lib64:" + self.binaryPath + "/usr/lib:" + self.binaryPath + "/usr/lib64:" + self.binaryPath + "/usr/local/lib:" + "/home/admin/isearch5_data/AliWS-1.4.0.0/usr/local/lib:" + "/usr/local/lib64:"
self.binPath = self.binaryPath + "/home/admin/sap/bin/:" + self.binaryPath + "/usr/local/bin/:" + \
self.binaryPath + "/usr/bin/:" + self.binaryPath + "/bin/:" + "/usr/local/bin:/usr/bin:/bin"
self.startCmdTemplate = "/bin/env HIPPO_DP2_SLAVE_PORT=19715"
if self.preload:
if self.preload == "asan":
self.startCmdTemplate += " LSAN_OPTIONS=\"suppressions=%s/usr/local/etc/sql/leak_suppression\" ASAN_OPTIONS=\"detect_odr_violation=0 abort_on_error=1 detect_leaks=1\"" % (
self.binaryPath)
elif self.preload == "llalloc":
self.startCmdTemplate += " LD_PRELOAD=%s/usr/local/lib64/libllalloc.so" % self.binaryPath
else:
self.startCmdTemplate += " LD_PRELOAD=%s" % self.preload
self.startCmdTemplate += " JAVA_HOME=/usr/local/java HADOOP_HOME=/usr/local/hadoop/hadoop CLASSPATH=/usr/local/hadoop/hadoop/etc/hadoop:/usr/local/hadoop/hadoop/share/hadoop/common/lib/*:/usr/local/hadoop/hadoop/share/hadoop/common/*:/usr/local/hadoop/hadoop/share/hadoop/hdfs:/usr/local/hadoop/hadoop/share/hadoop/hdfs/lib/*:/usr/local/hadoop/hadoop/share/hadoop/hdfs/*:/usr/local/hadoop/hadoop/share/hadoop/yarn/lib/*:/usr/local/hadoop/hadoop/share/hadoop/yarn/*:/usr/local/hadoop/hadoop/share/hadoop/mapreduce/lib/*:/usr/local/hadoop/hadoop/share/hadoop/mapreduce/*:/usr/local/hadoop/hadoop/contrib/capacity-scheduler/*.jar "
if self.dailyrunMode:
self.startCmdTemplate += " DAILY_RUN_MODE=true"
self.startCmdTemplate += " IS_TEST_MODE=true"
if self.disableCodeGen:
self.startCmdTemplate += " DISABLE_CODEGEN=true"
if self.disableTurbojet:
self.startCmdTemplate += " DISABLE_TURBOJET=true"
if self.enablePublishTableTopoInfo:
self.startCmdTemplate += " ENABLE_PUBLISH_TABLE_TOPO_INFO=true"
self.startCmdTemplate += " HIPPO_APP_INST_ROOT=" + self.binaryPath + \
" HIPPO_APP_WORKDIR=" + os.getcwd() + " TJ_RUNTIME_TEMP_DIR=" + self.binaryPath
self.startCmdTemplate += " PATH=$JAVA_HOME/bin:%s LD_LIBRARY_PATH=%s ha_sql --env roleType=%s -l %s -r %s -c %s --port arpc:%d --port http:%d --env httpPort=%d --env gigGrpcPort=0 --env serviceName=%s --env amonitorPath=%s/%s --env port=%d --env ip=%s --env userName=admin --env decodeUri=true --env haCompatible=true --env zoneName=%s --env roleName=%s_partition_%d --env partId=0 --env decisionLoopInterval=10000 --env dpThreadNum=1 --env loadThreadNum=4 --env load_biz_thread_num=4 --env kmonitorNormalSamplePeriod=1 --env naviPoolModeAsan=1 --env naviDisablePerf=1 --env WORKER_IDENTIFIER_FOR_CARBON= --env gigGrpcThreadNum=5 --env GRPC_CLIENT_CHANNEL_BACKUP_POLL_INTERVAL_MS=500 --env FAST_CLEAN_INC_VERSION=false --env navi_config_loader=%s"
if self.localBizService:
self.startCmdTemplate += " --env localBizService=true"
self.alogConfigPath = os.path.join(self.binaryPath, "usr/local/etc/sql/sql_alog.conf")
self.searchCfg = os.path.join(self.binaryPath, "usr/local/etc/sql/search_server.cfg")
self.qrsCfg = os.path.join(self.binaryPath, "usr/local/etc/sql/qrs_server.cfg")
self.config_loader = os.path.join(self.binaryPath,
"usr/local/lib/python/site-packages/sql/sql_config_loader.py")
self.ip = socket.gethostbyname(socket.gethostname())
self.gigInfos = {}
cwd = os.getcwd()
self.localSearchDir = os.path.join(cwd, "local_search_%d" % self.portStart)
if not os.path.exists(self.localSearchDir):
os.system("mkdir %s" % self.localSearchDir)
binarySymbol = "%s/binary" % (self.localSearchDir)
if not os.path.exists(binarySymbol):
try:
os.symlink(self.binaryPath, binarySymbol)
except OSError as e:
if e.errno == errno.EEXIST:
os.remove(binarySymbol)
os.symlink(self.binaryPath, binarySymbol)
else:
raise e
self.pidFile = os.path.join(self.localSearchDir, "pid")
self.portFile = os.path.join(self.localSearchDir, "ports")
self.qrsPortFile = os.path.join(cwd, 'qrs_port')
self.searcherPortFile = os.path.join(cwd, 'searcher_port')
self.start_time = datetime.now()
self.targetVersion = 1651870394